2024-09-21T17:06:45.248 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T17:06:45.266 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T17:06:45.342 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048 branch: wip-adk-testing-2024-09-21-1014 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7914048' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-09-21-1014 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 0ecbb435437226bc091a0c2d4549ba25bf724efe ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 0ecbb435437226bc091a0c2d4549ba25bf724efe selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-adk-testing-2024-09-21-1014 sha1: 0ecbb435437226bc091a0c2d4549ba25bf724efe owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 9557 sha1: 0ecbb435437226bc091a0c2d4549ba25bf724efe sleep_before_teardown: 0 subset: 1/20 suite: orch:cephadm suite_branch: wip-adk-testing-2024-09-21-1014 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_0ecbb435437226bc091a0c2d4549ba25bf724efe/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 0ecbb435437226bc091a0c2d4549ba25bf724efe targets: smithi018.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDS6iddN6gcV4v91W15fKroK5nI1CzUfr8GCCGpSpq4KtK5LLJhZVU8YJhVYPaPR1L6ovfEs4QXP/TPobEvatuM= smithi157.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD8psHl4Vm03YBoj4r+ryhNDN4z3XsC7cuyUkZesIXqYr1gAIAWk1Vd9a0+oa15K9/tyWP4qDPx4nAQfLbZ/OgI= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_16:57:11 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-21T17:06:45.342 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_0ecbb435437226bc091a0c2d4549ba25bf724efe/qa; will attempt to use it 2024-09-21T17:06:45.343 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_0ecbb435437226bc091a0c2d4549ba25bf724efe/qa/tasks 2024-09-21T17:06:45.343 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T17:06:45.345 INFO:teuthology.task.internal:Checking packages... 2024-09-21T17:06:45.365 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '0ecbb435437226bc091a0c2d4549ba25bf724efe' 2024-09-21T17:06:45.365 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T17:06:45.365 INFO:teuthology.packaging:ref: None 2024-09-21T17:06:45.366 INFO:teuthology.packaging:tag: None 2024-09-21T17:06:45.366 INFO:teuthology.packaging:branch: wip-adk-testing-2024-09-21-1014 2024-09-21T17:06:45.366 INFO:teuthology.packaging:sha1: 0ecbb435437226bc091a0c2d4549ba25bf724efe 2024-09-21T17:06:45.366 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-09-21-1014 2024-09-21T17:06:45.507 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5129.g0ecbb435 2024-09-21T17:06:45.509 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T17:06:45.515 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T17:06:45.515 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T17:06:45.522 INFO:teuthology.task.internal:Saving configuration 2024-09-21T17:06:45.535 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T17:06:45.542 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T17:06:45.566 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi018.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 17:01:02.199050', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDS6iddN6gcV4v91W15fKroK5nI1CzUfr8GCCGpSpq4KtK5LLJhZVU8YJhVYPaPR1L6ovfEs4QXP/TPobEvatuM='} 2024-09-21T17:06:45.589 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi157.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 17:01:02.200481', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD8psHl4Vm03YBoj4r+ryhNDN4z3XsC7cuyUkZesIXqYr1gAIAWk1Vd9a0+oa15K9/tyWP4qDPx4nAQfLbZ/OgI='} 2024-09-21T17:06:45.589 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T17:06:45.595 INFO:teuthology.task.internal:roles: ubuntu@smithi018.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-21T17:06:45.595 INFO:teuthology.task.internal:roles: ubuntu@smithi157.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-21T17:06:45.595 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T17:06:45.668 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f87ba914280>, signals=[15]) 2024-09-21T17:06:45.669 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T17:06:45.675 INFO:teuthology.task.internal:Opening connections... 2024-09-21T17:06:45.675 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:06:45.676 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:06:45.750 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi157.front.sepia.ceph.com 2024-09-21T17:06:45.751 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi157.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:06:45.822 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T17:06:45.829 DEBUG:teuthology.orchestra.run.smithi018:> uname -m 2024-09-21T17:06:45.847 INFO:teuthology.orchestra.run.smithi018.stdout:x86_64 2024-09-21T17:06:45.847 DEBUG:teuthology.orchestra.run.smithi018:> cat /etc/os-release 2024-09-21T17:06:45.905 INFO:teuthology.orchestra.run.smithi018.stdout:NAME="CentOS Stream" 2024-09-21T17:06:45.905 INFO:teuthology.orchestra.run.smithi018.stdout:VERSION="9" 2024-09-21T17:06:45.905 INFO:teuthology.orchestra.run.smithi018.stdout:ID="centos" 2024-09-21T17:06:45.905 INFO:teuthology.orchestra.run.smithi018.stdout:ID_LIKE="rhel fedora" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:VERSION_ID="9" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:PLATFORM_ID="platform:el9" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:ANSI_COLOR="0;31" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:LOGO="fedora-logo-icon" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:HOME_URL="https://centos.org/" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T17:06:45.906 INFO:teuthology.orchestra.run.smithi018.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T17:06:45.907 INFO:teuthology.lock.ops:Updating smithi018.front.sepia.ceph.com on lock server 2024-09-21T17:06:45.926 DEBUG:teuthology.orchestra.run.smithi157:> uname -m 2024-09-21T17:06:45.944 INFO:teuthology.orchestra.run.smithi157.stdout:x86_64 2024-09-21T17:06:45.944 DEBUG:teuthology.orchestra.run.smithi157:> cat /etc/os-release 2024-09-21T17:06:46.001 INFO:teuthology.orchestra.run.smithi157.stdout:NAME="CentOS Stream" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:VERSION="9" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:ID="centos" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:ID_LIKE="rhel fedora" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:VERSION_ID="9" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:PLATFORM_ID="platform:el9" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:ANSI_COLOR="0;31" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:LOGO="fedora-logo-icon" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:HOME_URL="https://centos.org/" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T17:06:46.002 INFO:teuthology.orchestra.run.smithi157.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T17:06:46.003 INFO:teuthology.orchestra.run.smithi157.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T17:06:46.003 INFO:teuthology.lock.ops:Updating smithi157.front.sepia.ceph.com on lock server 2024-09-21T17:06:46.026 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T17:06:46.040 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T17:06:46.045 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T17:06:46.045 DEBUG:teuthology.orchestra.run.smithi018:> test '!' -e /home/ubuntu/cephtest 2024-09-21T17:06:46.047 DEBUG:teuthology.orchestra.run.smithi157:> test '!' -e /home/ubuntu/cephtest 2024-09-21T17:06:46.063 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T17:06:46.077 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T17:06:46.077 DEBUG:teuthology.orchestra.run.smithi018:> test -z $(ls -A /var/lib/ceph) 2024-09-21T17:06:46.105 DEBUG:teuthology.orchestra.run.smithi157:> test -z $(ls -A /var/lib/ceph) 2024-09-21T17:06:46.196 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T17:06:46.255 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T17:06:46.269 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T17:06:46.269 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T17:06:46.269 DEBUG:teuthology.orchestra.run.smithi018:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T17:06:46.270 DEBUG:teuthology.orchestra.run.smithi157:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T17:06:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T17:06:46.286 DEBUG:teuthology.orchestra.run.smithi018:> uname -r 2024-09-21T17:06:46.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T17:06:46.287 DEBUG:teuthology.orchestra.run.smithi157:> uname -r 2024-09-21T17:06:46.344 INFO:teuthology.orchestra.run.smithi018.stdout:5.14.0-503.el9.x86_64 2024-09-21T17:06:46.344 INFO:teuthology.task.kernel:Running kernel on smithi018: 5.14.0-503.el9.x86_64 2024-09-21T17:06:46.344 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum install -y kernel 2024-09-21T17:06:46.345 INFO:teuthology.orchestra.run.smithi157.stdout:5.14.0-503.el9.x86_64 2024-09-21T17:06:46.345 INFO:teuthology.task.kernel:Running kernel on smithi157: 5.14.0-503.el9.x86_64 2024-09-21T17:06:46.345 DEBUG:teuthology.orchestra.run.smithi157:> sudo yum install -y kernel 2024-09-21T17:06:48.969 INFO:teuthology.orchestra.run.smithi157.stdout:CentOS Stream 9 - BaseOS 9.2 MB/s | 8.3 MB 00:00 2024-09-21T17:06:49.227 INFO:teuthology.orchestra.run.smithi018.stdout:CentOS Stream 9 - BaseOS 7.5 MB/s | 8.3 MB 00:01 2024-09-21T17:06:53.070 INFO:teuthology.orchestra.run.smithi018.stdout:CentOS Stream 9 - AppStream 7.5 MB/s | 20 MB 00:02 2024-09-21T17:06:57.661 INFO:teuthology.orchestra.run.smithi157.stdout:CentOS Stream 9 - AppStream 2.7 MB/s | 20 MB 00:07 2024-09-21T17:06:57.771 INFO:teuthology.orchestra.run.smithi018.stdout:CentOS Stream 9 - CRB 9.3 MB/s | 6.5 MB 00:00 2024-09-21T17:06:59.563 INFO:teuthology.orchestra.run.smithi018.stdout:CentOS Stream 9 - Extras packages 36 kB/s | 19 kB 00:00 2024-09-21T17:07:00.375 INFO:teuthology.orchestra.run.smithi018.stdout:Extra Packages for Enterprise Linux 33 MB/s | 23 MB 00:00 2024-09-21T17:07:01.828 INFO:teuthology.orchestra.run.smithi157.stdout:CentOS Stream 9 - CRB 16 MB/s | 6.5 MB 00:00 2024-09-21T17:07:03.704 INFO:teuthology.orchestra.run.smithi157.stdout:CentOS Stream 9 - Extras packages 37 kB/s | 19 kB 00:00 2024-09-21T17:07:04.883 INFO:teuthology.orchestra.run.smithi157.stdout:Extra Packages for Enterprise Linux 21 MB/s | 23 MB 00:01 2024-09-21T17:07:05.973 INFO:teuthology.orchestra.run.smithi018.stdout:lab-extras 42 kB/s | 1.7 kB 00:00 2024-09-21T17:07:07.921 INFO:teuthology.orchestra.run.smithi018.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T17:07:07.921 INFO:teuthology.orchestra.run.smithi018.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T17:07:08.015 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2024-09-21T17:07:08.020 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2024-09-21T17:07:08.020 INFO:teuthology.orchestra.run.smithi018.stdout: Package Architecture Version Repository Size 2024-09-21T17:07:08.020 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2024-09-21T17:07:08.020 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2024-09-21T17:07:08.020 INFO:teuthology.orchestra.run.smithi018.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T17:07:08.020 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2024-09-21T17:07:08.021 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T17:07:08.021 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T17:07:08.021 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T17:07:08.021 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:07:08.021 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2024-09-21T17:07:08.021 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2024-09-21T17:07:08.021 INFO:teuthology.orchestra.run.smithi018.stdout:Install 4 Packages 2024-09-21T17:07:08.021 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:07:08.022 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 79 M 2024-09-21T17:07:08.022 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 126 M 2024-09-21T17:07:08.022 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2024-09-21T17:07:08.166 INFO:teuthology.orchestra.run.smithi018.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 379 kB/s | 36 kB 00:00 2024-09-21T17:07:08.616 INFO:teuthology.orchestra.run.smithi018.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 29 MB/s | 16 MB 00:00 2024-09-21T17:07:09.050 INFO:teuthology.orchestra.run.smithi018.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 35 MB/s | 35 MB 00:00 2024-09-21T17:07:09.342 INFO:teuthology.orchestra.run.smithi018.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-21T17:07:09.342 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2024-09-21T17:07:09.342 INFO:teuthology.orchestra.run.smithi018.stdout:Total 60 MB/s | 79 MB 00:01 2024-09-21T17:07:10.054 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2024-09-21T17:07:10.305 INFO:teuthology.orchestra.run.smithi157.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-21T17:07:10.456 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2024-09-21T17:07:10.457 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2024-09-21T17:07:11.466 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2024-09-21T17:07:11.466 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2024-09-21T17:07:12.212 INFO:teuthology.orchestra.run.smithi157.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T17:07:12.212 INFO:teuthology.orchestra.run.smithi157.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T17:07:12.302 INFO:teuthology.orchestra.run.smithi157.stdout:Dependencies resolved. 2024-09-21T17:07:12.313 INFO:teuthology.orchestra.run.smithi157.stdout:================================================================================ 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout: Package Architecture Version Repository Size 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout:================================================================================ 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout:Installing: 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout:Installing dependencies: 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout:Transaction Summary 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout:================================================================================ 2024-09-21T17:07:12.314 INFO:teuthology.orchestra.run.smithi157.stdout:Install 4 Packages 2024-09-21T17:07:12.315 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:07:12.315 INFO:teuthology.orchestra.run.smithi157.stdout:Total download size: 79 M 2024-09-21T17:07:12.315 INFO:teuthology.orchestra.run.smithi157.stdout:Installed size: 126 M 2024-09-21T17:07:12.315 INFO:teuthology.orchestra.run.smithi157.stdout:Downloading Packages: 2024-09-21T17:07:12.923 INFO:teuthology.orchestra.run.smithi157.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 64 kB/s | 36 kB 00:00 2024-09-21T17:07:14.044 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2024-09-21T17:07:14.719 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T17:07:14.809 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T17:07:15.015 INFO:teuthology.orchestra.run.smithi157.stdout:(2/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 13 MB/s | 35 MB 00:02 2024-09-21T17:07:15.332 INFO:teuthology.orchestra.run.smithi157.stdout:(3/4): kernel-core-5.14.0-508.el9.x86_64.rpm 5.3 MB/s | 16 MB 00:02 2024-09-21T17:07:16.982 INFO:teuthology.orchestra.run.smithi157.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 7.0 MB/s | 29 MB 00:04 2024-09-21T17:07:16.982 INFO:teuthology.orchestra.run.smithi157.stdout:-------------------------------------------------------------------------------- 2024-09-21T17:07:16.983 INFO:teuthology.orchestra.run.smithi157.stdout:Total 17 MB/s | 79 MB 00:04 2024-09-21T17:07:17.568 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T17:07:17.672 INFO:teuthology.orchestra.run.smithi157.stdout:Running transaction check 2024-09-21T17:07:17.695 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T17:07:18.029 INFO:teuthology.orchestra.run.smithi157.stdout:Transaction check succeeded. 2024-09-21T17:07:18.030 INFO:teuthology.orchestra.run.smithi157.stdout:Running transaction test 2024-09-21T17:07:18.988 INFO:teuthology.orchestra.run.smithi157.stdout:Transaction test succeeded. 2024-09-21T17:07:18.988 INFO:teuthology.orchestra.run.smithi157.stdout:Running transaction 2024-09-21T17:07:21.429 INFO:teuthology.orchestra.run.smithi157.stdout: Preparing : 1/1 2024-09-21T17:07:22.161 INFO:teuthology.orchestra.run.smithi157.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T17:07:22.249 INFO:teuthology.orchestra.run.smithi157.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T17:07:22.815 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T17:07:22.863 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:07:25.035 INFO:teuthology.orchestra.run.smithi157.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T17:07:25.162 INFO:teuthology.orchestra.run.smithi157.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T17:07:28.004 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:07:29.979 INFO:teuthology.orchestra.run.smithi157.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T17:07:30.021 INFO:teuthology.orchestra.run.smithi157.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:07:34.868 INFO:teuthology.orchestra.run.smithi157.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:07.224 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:07.265 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:07.580 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:07.580 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T17:08:07.580 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T17:08:07.580 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T17:08:08.390 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:08.390 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:08:08.390 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2024-09-21T17:08:08.390 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T17:08:08.390 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T17:08:08.390 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T17:08:08.391 INFO:teuthology.orchestra.run.smithi018.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T17:08:08.391 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:08:08.391 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2024-09-21T17:08:08.751 DEBUG:teuthology.orchestra.run.smithi018:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T17:08:08.834 INFO:teuthology.orchestra.run.smithi018.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T17:08:08.834 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T17:08:08.834 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T17:08:08.834 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T17:08:08.835 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T17:08:08.835 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-21T17:08:08.835 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi018.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T17:08:08.835 DEBUG:teuthology.orchestra.run.smithi018:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T17:08:08.894 INFO:teuthology.orchestra.run.smithi018.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T17:08:08.894 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T17:08:08.894 DEBUG:teuthology.orchestra.run.smithi018:> sudo rpm -qi grub2-tools 2024-09-21T17:08:08.934 INFO:teuthology.orchestra.run.smithi018.stdout:Name : grub2-tools 2024-09-21T17:08:08.934 INFO:teuthology.orchestra.run.smithi018.stdout:Epoch : 1 2024-09-21T17:08:08.934 INFO:teuthology.orchestra.run.smithi018.stdout:Version : 2.06 2024-09-21T17:08:08.934 INFO:teuthology.orchestra.run.smithi018.stdout:Release : 82.el9 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Architecture: x86_64 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Group : Unspecified 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Size : 8274670 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:License : GPLv3+ 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Packager : builder@centos.org 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Vendor : CentOS 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Summary : Support tools for GRUB. 2024-09-21T17:08:08.935 INFO:teuthology.orchestra.run.smithi018.stdout:Description : 2024-09-21T17:08:08.936 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:08:08.936 INFO:teuthology.orchestra.run.smithi018.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T17:08:08.936 INFO:teuthology.orchestra.run.smithi018.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T17:08:08.936 INFO:teuthology.orchestra.run.smithi018.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T17:08:08.936 INFO:teuthology.orchestra.run.smithi018.stdout:hardware devices. 2024-09-21T17:08:08.936 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:08:08.936 INFO:teuthology.orchestra.run.smithi018.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T17:08:08.937 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T17:08:08.937 INFO:teuthology.task.kernel:Updating grub on smithi018 to boot 5.14.0-508.el9.x86_64 2024-09-21T17:08:08.937 DEBUG:teuthology.orchestra.run.smithi018:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T17:08:09.568 INFO:teuthology.orchestra.run.smithi018.stderr:Generating grub configuration file ... 2024-09-21T17:08:10.937 INFO:teuthology.orchestra.run.smithi018.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T17:08:10.966 INFO:teuthology.orchestra.run.smithi018.stderr:done 2024-09-21T17:08:10.968 DEBUG:teuthology.orchestra.run.smithi018:> mktemp 2024-09-21T17:08:10.983 INFO:teuthology.orchestra.run.smithi018.stdout:/tmp/tmp.6kxScnETTK 2024-09-21T17:08:10.983 DEBUG:teuthology.orchestra.run.smithi018:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.6kxScnETTK 2024-09-21T17:08:11.056 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod 0666 /tmp/tmp.6kxScnETTK 2024-09-21T17:08:11.196 DEBUG:teuthology.orchestra.remote:smithi018:/tmp/tmp.6kxScnETTK is 6KB 2024-09-21T17:08:11.217 DEBUG:teuthology.orchestra.run.smithi018:> rm -fr /tmp/tmp.6kxScnETTK 2024-09-21T17:08:11.231 DEBUG:teuthology.orchestra.run.smithi018:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T17:08:11.293 INFO:teuthology.orchestra.run.smithi018.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T17:08:11.293 INFO:teuthology.orchestra.run.smithi018.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T17:08:11.293 INFO:teuthology.orchestra.run.smithi018.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T17:08:11.293 INFO:teuthology.orchestra.run.smithi018.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T17:08:11.294 DEBUG:teuthology.orchestra.run.smithi018:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T17:08:11.461 DEBUG:teuthology.orchestra.run.smithi018:> sudo shutdown -r now 2024-09-21T17:08:13.321 INFO:teuthology.orchestra.run.smithi157.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:13.364 INFO:teuthology.orchestra.run.smithi157.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:13.640 INFO:teuthology.orchestra.run.smithi157.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:13.640 INFO:teuthology.orchestra.run.smithi157.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T17:08:13.640 INFO:teuthology.orchestra.run.smithi157.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T17:08:13.640 INFO:teuthology.orchestra.run.smithi157.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout:Installed: 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:08:14.467 INFO:teuthology.orchestra.run.smithi157.stdout:Complete! 2024-09-21T17:08:14.844 DEBUG:teuthology.orchestra.run.smithi157:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T17:08:14.938 INFO:teuthology.orchestra.run.smithi157.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T17:08:14.938 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T17:08:14.938 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T17:08:14.938 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T17:08:14.938 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T17:08:14.938 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-21T17:08:14.938 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi157.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T17:08:14.938 DEBUG:teuthology.orchestra.run.smithi157:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T17:08:14.997 INFO:teuthology.orchestra.run.smithi157.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T17:08:14.997 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T17:08:14.997 DEBUG:teuthology.orchestra.run.smithi157:> sudo rpm -qi grub2-tools 2024-09-21T17:08:15.029 INFO:teuthology.orchestra.run.smithi157.stdout:Name : grub2-tools 2024-09-21T17:08:15.029 INFO:teuthology.orchestra.run.smithi157.stdout:Epoch : 1 2024-09-21T17:08:15.029 INFO:teuthology.orchestra.run.smithi157.stdout:Version : 2.06 2024-09-21T17:08:15.029 INFO:teuthology.orchestra.run.smithi157.stdout:Release : 82.el9 2024-09-21T17:08:15.029 INFO:teuthology.orchestra.run.smithi157.stdout:Architecture: x86_64 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Group : Unspecified 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Size : 8274670 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:License : GPLv3+ 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Packager : builder@centos.org 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Vendor : CentOS 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Summary : Support tools for GRUB. 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout:Description : 2024-09-21T17:08:15.030 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:08:15.031 INFO:teuthology.orchestra.run.smithi157.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T17:08:15.031 INFO:teuthology.orchestra.run.smithi157.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T17:08:15.031 INFO:teuthology.orchestra.run.smithi157.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T17:08:15.031 INFO:teuthology.orchestra.run.smithi157.stdout:hardware devices. 2024-09-21T17:08:15.031 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:08:15.031 INFO:teuthology.orchestra.run.smithi157.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T17:08:15.032 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T17:08:15.032 INFO:teuthology.task.kernel:Updating grub on smithi157 to boot 5.14.0-508.el9.x86_64 2024-09-21T17:08:15.032 DEBUG:teuthology.orchestra.run.smithi157:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T17:08:15.713 INFO:teuthology.orchestra.run.smithi157.stderr:Generating grub configuration file ... 2024-09-21T17:08:17.025 INFO:teuthology.orchestra.run.smithi157.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T17:08:17.049 INFO:teuthology.orchestra.run.smithi157.stderr:done 2024-09-21T17:08:17.052 DEBUG:teuthology.orchestra.run.smithi157:> mktemp 2024-09-21T17:08:17.067 INFO:teuthology.orchestra.run.smithi157.stdout:/tmp/tmp.g4xbwQgrd5 2024-09-21T17:08:17.067 DEBUG:teuthology.orchestra.run.smithi157:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.g4xbwQgrd5 2024-09-21T17:08:17.134 DEBUG:teuthology.orchestra.run.smithi157:> sudo chmod 0666 /tmp/tmp.g4xbwQgrd5 2024-09-21T17:08:17.280 DEBUG:teuthology.orchestra.remote:smithi157:/tmp/tmp.g4xbwQgrd5 is 6KB 2024-09-21T17:08:17.328 DEBUG:teuthology.orchestra.run.smithi157:> rm -fr /tmp/tmp.g4xbwQgrd5 2024-09-21T17:08:17.343 DEBUG:teuthology.orchestra.run.smithi157:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T17:08:17.406 INFO:teuthology.orchestra.run.smithi157.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T17:08:17.406 INFO:teuthology.orchestra.run.smithi157.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T17:08:17.406 INFO:teuthology.orchestra.run.smithi157.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T17:08:17.406 INFO:teuthology.orchestra.run.smithi157.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T17:08:17.407 DEBUG:teuthology.orchestra.run.smithi157:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T17:08:17.554 DEBUG:teuthology.orchestra.run.smithi157:> sudo shutdown -r now 2024-09-21T17:08:41.486 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-21T17:08:41.487 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-21T17:08:41.488 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:08:47.562 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-21T17:08:47.563 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi157.front.sepia.ceph.com' 2024-09-21T17:08:47.563 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi157.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:09:00.015 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.18 2024-09-21T17:09:05.903 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.157 2024-09-21T17:09:09.018 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-21T17:09:09.019 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:09:14.909 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi157.front.sepia.ceph.com' 2024-09-21T17:09:14.910 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi157.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:09:17.967 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.157 2024-09-21T17:09:29.971 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi157.front.sepia.ceph.com' 2024-09-21T17:09:29.972 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi157.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:09:33.039 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.157 2024-09-21T17:09:48.050 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi157.front.sepia.ceph.com' 2024-09-21T17:09:48.051 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi157.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:10:06.575 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.157 2024-09-21T17:10:09.024 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T17:10:21.036 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-21T17:10:21.037 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:10:21.338 DEBUG:teuthology.orchestra.run.smithi018:> true 2024-09-21T17:10:21.695 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-21T17:10:21.695 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T17:10:21.696 DEBUG:teuthology.orchestra.run.smithi018:> uname -r 2024-09-21T17:10:21.712 INFO:teuthology.orchestra.run.smithi018.stdout:5.14.0-508.el9.x86_64 2024-09-21T17:10:21.712 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T17:10:21.712 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T17:10:21.713 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T17:10:22.714 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-21T17:10:22.714 DEBUG:teuthology.orchestra.run.smithi018:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T17:10:22.782 INFO:teuthology.orchestra.run.smithi018.stdout:ttyS1 2024-09-21T17:10:22.802 DEBUG:teuthology.parallel:result is None 2024-09-21T17:10:24.576 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi157.front.sepia.ceph.com' 2024-09-21T17:10:24.576 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi157.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:10:24.835 DEBUG:teuthology.orchestra.run.smithi157:> true 2024-09-21T17:10:25.216 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi157.front.sepia.ceph.com' 2024-09-21T17:10:25.217 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T17:10:25.217 DEBUG:teuthology.orchestra.run.smithi157:> uname -r 2024-09-21T17:10:25.231 INFO:teuthology.orchestra.run.smithi157.stdout:5.14.0-508.el9.x86_64 2024-09-21T17:10:25.232 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T17:10:25.232 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T17:10:25.232 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T17:10:26.233 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-21T17:10:26.233 DEBUG:teuthology.orchestra.run.smithi157:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T17:10:26.319 INFO:teuthology.orchestra.run.smithi157.stdout:ttyS1 2024-09-21T17:10:26.336 DEBUG:teuthology.parallel:result is None 2024-09-21T17:10:26.336 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T17:10:26.345 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T17:10:26.345 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T17:10:26.348 DEBUG:teuthology.orchestra.run.smithi157:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T17:10:26.364 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T17:10:26.372 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T17:10:26.383 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T17:10:26.383 DEBUG:teuthology.orchestra.run.smithi018:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T17:10:26.407 DEBUG:teuthology.orchestra.run.smithi157:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T17:10:26.449 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T17:10:26.456 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T17:10:26.456 DEBUG:teuthology.orchestra.run.smithi018:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T17:10:26.477 DEBUG:teuthology.orchestra.run.smithi157:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T17:10:26.511 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T17:10:26.516 INFO:teuthology.orchestra.run.smithi157.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T17:10:26.523 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T17:10:26.526 INFO:teuthology.orchestra.run.smithi157.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T17:10:26.528 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T17:10:26.534 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T17:10:26.535 DEBUG:teuthology.orchestra.run.smithi018:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T17:10:26.568 DEBUG:teuthology.orchestra.run.smithi157:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T17:10:26.594 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T17:10:26.603 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T17:10:26.603 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T17:10:26.637 DEBUG:teuthology.orchestra.run.smithi157:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T17:10:26.653 DEBUG:teuthology.orchestra.run.smithi018:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T17:10:26.736 DEBUG:teuthology.orchestra.run.smithi018:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T17:10:26.784 DEBUG:teuthology.orchestra.run.smithi018:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T17:10:26.868 DEBUG:teuthology.orchestra.run.smithi018:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T17:10:26.909 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:10:26.910 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T17:10:26.976 DEBUG:teuthology.orchestra.run.smithi157:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T17:10:27.053 DEBUG:teuthology.orchestra.run.smithi157:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T17:10:27.112 DEBUG:teuthology.orchestra.run.smithi157:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T17:10:27.189 DEBUG:teuthology.orchestra.run.smithi157:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T17:10:27.255 DEBUG:teuthology.orchestra.run.smithi157:> set -ex 2024-09-21T17:10:27.255 DEBUG:teuthology.orchestra.run.smithi157:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T17:10:27.295 DEBUG:teuthology.orchestra.run.smithi018:> sudo service rsyslog restart 2024-09-21T17:10:27.298 DEBUG:teuthology.orchestra.run.smithi157:> sudo service rsyslog restart 2024-09-21T17:10:27.341 INFO:teuthology.orchestra.run.smithi018.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T17:10:27.423 INFO:teuthology.orchestra.run.smithi157.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T17:10:27.896 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T17:10:27.905 INFO:teuthology.task.internal:Starting timer... 2024-09-21T17:10:27.905 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T17:10:27.967 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T17:10:27.979 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T17:10:27.979 DEBUG:teuthology.orchestra.run.smithi018:> sudo service auditd rotate 2024-09-21T17:10:28.042 INFO:teuthology.orchestra.run.smithi018.stdout:Rotating logs: 2024-09-21T17:10:28.044 DEBUG:teuthology.orchestra.run.smithi157:> sudo service auditd rotate 2024-09-21T17:10:28.199 INFO:teuthology.orchestra.run.smithi157.stdout:Rotating logs: 2024-09-21T17:10:28.201 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T17:10:28.201 DEBUG:teuthology.orchestra.run.smithi018:> /usr/sbin/getenforce 2024-09-21T17:10:28.234 INFO:teuthology.orchestra.run.smithi018.stdout:Permissive 2024-09-21T17:10:28.234 DEBUG:teuthology.orchestra.run.smithi157:> /usr/sbin/getenforce 2024-09-21T17:10:28.277 INFO:teuthology.orchestra.run.smithi157.stdout:Permissive 2024-09-21T17:10:28.277 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi018.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi157.front.sepia.ceph.com': 'permissive'} 2024-09-21T17:10:28.278 DEBUG:teuthology.orchestra.run.smithi018:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T17:10:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T17:10:28.307 DEBUG:teuthology.orchestra.run.smithi157:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T17:10:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T17:10:28.334 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T17:10:28.334 DEBUG:teuthology.orchestra.run.smithi018:> sudo /usr/sbin/setenforce permissive 2024-09-21T17:10:28.383 DEBUG:teuthology.orchestra.run.smithi157:> sudo /usr/sbin/setenforce permissive 2024-09-21T17:10:28.414 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T17:10:28.429 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T17:10:28.439 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T17:10:28.440 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi018.front.sepia.ceph.com,smithi157.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T17:14:12.311 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi018.front.sepia.ceph.com'), Remote(name='ubuntu@smithi157.front.sepia.ceph.com')] 2024-09-21T17:14:12.312 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-21T17:14:12.313 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:14:12.389 DEBUG:teuthology.orchestra.run.smithi018:> true 2024-09-21T17:14:12.470 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi018.front.sepia.ceph.com' 2024-09-21T17:14:12.470 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi157.front.sepia.ceph.com' 2024-09-21T17:14:12.471 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi157.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T17:14:12.543 DEBUG:teuthology.orchestra.run.smithi157:> true 2024-09-21T17:14:12.618 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi157.front.sepia.ceph.com' 2024-09-21T17:14:12.618 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T17:14:12.635 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T17:14:12.635 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T17:14:12.635 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T17:14:12.638 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T17:14:12.638 DEBUG:teuthology.orchestra.run.smithi157:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T17:14:12.667 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T17:14:12.682 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T17:14:12.689 INFO:teuthology.orchestra.run.smithi157.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T17:14:12.702 INFO:teuthology.orchestra.run.smithi157.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T17:14:12.708 INFO:teuthology.orchestra.run.smithi018.stderr:sudo: ntpd: command not found 2024-09-21T17:14:12.720 INFO:teuthology.orchestra.run.smithi018.stdout:506 Cannot talk to daemon 2024-09-21T17:14:12.727 INFO:teuthology.orchestra.run.smithi157.stderr:sudo: ntpd: command not found 2024-09-21T17:14:12.735 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T17:14:12.738 INFO:teuthology.orchestra.run.smithi157.stdout:506 Cannot talk to daemon 2024-09-21T17:14:12.750 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T17:14:12.752 INFO:teuthology.orchestra.run.smithi157.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T17:14:12.767 INFO:teuthology.orchestra.run.smithi157.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T17:14:12.805 INFO:teuthology.orchestra.run.smithi018.stderr:bash: line 1: ntpq: command not found 2024-09-21T17:14:12.809 INFO:teuthology.orchestra.run.smithi018.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T17:14:12.809 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================== 2024-09-21T17:14:12.809 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:14:12.809 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:14:12.809 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:14:12.809 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:14:12.820 INFO:teuthology.orchestra.run.smithi157.stderr:bash: line 1: ntpq: command not found 2024-09-21T17:14:12.823 INFO:teuthology.orchestra.run.smithi157.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T17:14:12.824 INFO:teuthology.orchestra.run.smithi157.stdout:=============================================================================== 2024-09-21T17:14:12.824 INFO:teuthology.orchestra.run.smithi157.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:14:12.824 INFO:teuthology.orchestra.run.smithi157.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:14:12.824 INFO:teuthology.orchestra.run.smithi157.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:14:12.824 INFO:teuthology.orchestra.run.smithi157.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:14:12.825 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T17:14:12.908 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '0ecbb435437226bc091a0c2d4549ba25bf724efe'} 2024-09-21T17:14:12.908 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-21T17:14:12.908 INFO:tasks.cephadm:Cluster fsid is ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:12.908 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T17:14:12.908 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-21T17:14:12.909 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi018': '172.21.15.18', 'mon.smithi157': '172.21.15.157'} 2024-09-21T17:14:12.909 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T17:14:12.909 DEBUG:teuthology.orchestra.run.smithi018:> sudo hostname $(hostname -s) 2024-09-21T17:14:12.936 DEBUG:teuthology.orchestra.run.smithi157:> sudo hostname $(hostname -s) 2024-09-21T17:14:12.962 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-21T17:14:12.962 DEBUG:teuthology.orchestra.run.smithi018:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T17:14:13.209 INFO:teuthology.orchestra.run.smithi018.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 21 17:14 /home/ubuntu/cephtest/cephadm 2024-09-21T17:14:13.209 DEBUG:teuthology.orchestra.run.smithi157:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T17:14:13.334 INFO:teuthology.orchestra.run.smithi157.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 21 17:14 /home/ubuntu/cephtest/cephadm 2024-09-21T17:14:13.335 DEBUG:teuthology.orchestra.run.smithi018:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T17:14:13.352 DEBUG:teuthology.orchestra.run.smithi157:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T17:14:13.377 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-21T17:14:13.378 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-21T17:14:13.395 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-21T17:14:15.069 INFO:teuthology.orchestra.run.smithi157.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T17:14:15.083 INFO:teuthology.orchestra.run.smithi018.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T17:14:27.310 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:14:27.310 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-21T17:14:27.310 INFO:teuthology.orchestra.run.smithi018.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-21T17:14:27.310 INFO:teuthology.orchestra.run.smithi018.stdout: "repo_digests": [ 2024-09-21T17:14:27.310 INFO:teuthology.orchestra.run.smithi018.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-21T17:14:27.310 INFO:teuthology.orchestra.run.smithi018.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-21T17:14:27.310 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2024-09-21T17:14:27.311 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:14:27.733 INFO:teuthology.orchestra.run.smithi157.stdout:{ 2024-09-21T17:14:27.733 INFO:teuthology.orchestra.run.smithi157.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-21T17:14:27.734 INFO:teuthology.orchestra.run.smithi157.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-21T17:14:27.734 INFO:teuthology.orchestra.run.smithi157.stdout: "repo_digests": [ 2024-09-21T17:14:27.734 INFO:teuthology.orchestra.run.smithi157.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-21T17:14:27.734 INFO:teuthology.orchestra.run.smithi157.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-21T17:14:27.734 INFO:teuthology.orchestra.run.smithi157.stdout: ] 2024-09-21T17:14:27.734 INFO:teuthology.orchestra.run.smithi157.stdout:} 2024-09-21T17:14:27.751 DEBUG:teuthology.orchestra.run.smithi018:> sudo mkdir -p /etc/ceph 2024-09-21T17:14:27.786 DEBUG:teuthology.orchestra.run.smithi157:> sudo mkdir -p /etc/ceph 2024-09-21T17:14:27.814 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod 777 /etc/ceph 2024-09-21T17:14:27.856 DEBUG:teuthology.orchestra.run.smithi157:> sudo chmod 777 /etc/ceph 2024-09-21T17:14:27.880 INFO:tasks.cephadm:Writing seed config... 2024-09-21T17:14:27.881 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T17:14:27.881 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T17:14:27.881 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T17:14:27.881 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T17:14:27.881 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T17:14:27.881 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T17:14:27.881 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T17:14:27.881 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T17:14:27.882 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:14:27.883 DEBUG:teuthology.orchestra.run.smithi018:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T17:14:27.913 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ec44ef70-783c-11ef-baf6-efdc52797490 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T17:14:27.914 DEBUG:teuthology.orchestra.run.smithi018:mon.smithi018> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018.service 2024-09-21T17:14:27.956 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T17:14:27.956 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid ec44ef70-783c-11ef-baf6-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.18 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T17:14:28.160 INFO:teuthology.orchestra.run.smithi018.stderr:-------------------------------------------------------------------------------- 2024-09-21T17:14:28.160 INFO:teuthology.orchestra.run.smithi018.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'ec44ef70-783c-11ef-baf6-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.18', '--skip-admin-label'] 2024-09-21T17:14:28.200 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: 5.2.2 2024-09-21T17:14:28.202 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying podman|docker is present... 2024-09-21T17:14:28.229 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: 5.2.2 2024-09-21T17:14:28.232 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying lvm2 is present... 2024-09-21T17:14:28.232 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying time synchronization is in place... 2024-09-21T17:14:28.241 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T17:14:28.249 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-21T17:14:28.257 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: enabled 2024-09-21T17:14:28.265 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: active 2024-09-21T17:14:28.265 INFO:teuthology.orchestra.run.smithi018.stderr:Unit chronyd.service is enabled and running 2024-09-21T17:14:28.265 INFO:teuthology.orchestra.run.smithi018.stderr:Repeating the final host check... 2024-09-21T17:14:28.294 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: 5.2.2 2024-09-21T17:14:28.296 INFO:teuthology.orchestra.run.smithi018.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-21T17:14:28.296 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl is present 2024-09-21T17:14:28.296 INFO:teuthology.orchestra.run.smithi018.stderr:lvcreate is present 2024-09-21T17:14:28.303 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T17:14:28.310 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-21T17:14:28.318 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: enabled 2024-09-21T17:14:28.325 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: active 2024-09-21T17:14:28.326 INFO:teuthology.orchestra.run.smithi018.stderr:Unit chronyd.service is enabled and running 2024-09-21T17:14:28.327 INFO:teuthology.orchestra.run.smithi018.stderr:Host looks OK 2024-09-21T17:14:28.327 INFO:teuthology.orchestra.run.smithi018.stderr:Cluster fsid: ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:28.327 INFO:teuthology.orchestra.run.smithi018.stderr:Acquiring lock 139928050583440 on /run/cephadm/ec44ef70-783c-11ef-baf6-efdc52797490.lock 2024-09-21T17:14:28.327 INFO:teuthology.orchestra.run.smithi018.stderr:Lock 139928050583440 acquired on /run/cephadm/ec44ef70-783c-11ef-baf6-efdc52797490.lock 2024-09-21T17:14:28.327 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying IP 172.21.15.18 port 3300 ... 2024-09-21T17:14:28.327 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying IP 172.21.15.18 port 6789 ... 2024-09-21T17:14:28.328 INFO:teuthology.orchestra.run.smithi018.stderr:Base mon IP is 172.21.15.18, final addrv is [v2:172.21.15.18:3300,v1:172.21.15.18:6789] 2024-09-21T17:14:28.330 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.18 metric 100 2024-09-21T17:14:28.330 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.18 metric 100 2024-09-21T17:14:28.333 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T17:14:28.333 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-21T17:14:28.333 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1777sec hoplimit 64 pref medium 2024-09-21T17:14:28.336 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T17:14:28.336 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-21T17:14:28.336 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-21T17:14:28.336 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-21T17:14:28.336 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:1362/64 scope link 2024-09-21T17:14:28.336 INFO:teuthology.orchestra.run.smithi018.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-21T17:14:28.337 INFO:teuthology.orchestra.run.smithi018.stderr:Mon IP `172.21.15.18` is in CIDR network `172.21.0.0/20` 2024-09-21T17:14:28.337 INFO:teuthology.orchestra.run.smithi018.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T17:14:28.338 INFO:teuthology.orchestra.run.smithi018.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T17:14:28.459 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-21T17:14:28.862 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Getting image source signatures 2024-09-21T17:14:28.862 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-21T17:14:28.862 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-21T17:14:28.862 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-21T17:14:28.863 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-21T17:14:28.863 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-21T17:14:28.863 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-21T17:14:28.987 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: Writing manifest to image destination 2024-09-21T17:14:29.106 INFO:teuthology.orchestra.run.smithi018.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-21T17:14:29.528 INFO:teuthology.orchestra.run.smithi018.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-21T17:14:30.056 INFO:teuthology.orchestra.run.smithi018.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-21T17:14:30.056 INFO:teuthology.orchestra.run.smithi018.stderr:Extracting ceph user uid/gid from container image... 2024-09-21T17:14:31.872 INFO:teuthology.orchestra.run.smithi018.stderr:stat: 167 167 2024-09-21T17:14:37.498 INFO:teuthology.orchestra.run.smithi018.stderr:Creating initial keys... 2024-09-21T17:14:40.942 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph-authtool: AQAA/+5mg+Z/FRAA3Wji9eRlUr5ABPM8bu9mKg== 2024-09-21T17:14:41.822 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph-authtool: AQAB/+5mQU+ELBAA2va7O35ERTT9oCRlQ1AUjQ== 2024-09-21T17:14:42.711 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph-authtool: AQAC/+5mbz7oJRAAKbL6AaaNY4iTNWlgTHNYJQ== 2024-09-21T17:14:43.486 INFO:teuthology.orchestra.run.smithi018.stderr:Creating initial monmap... 2024-09-21T17:14:43.876 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T17:14:43.876 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-21T17:14:43.876 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:43.876 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T17:14:44.332 INFO:teuthology.orchestra.run.smithi018.stderr:monmaptool for smithi018 [v2:172.21.15.18:3300,v1:172.21.15.18:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T17:14:44.333 INFO:teuthology.orchestra.run.smithi018.stderr:setting min_mon_release = octopus 2024-09-21T17:14:44.333 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: set fsid to ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:44.333 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T17:14:44.333 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:14:44.333 INFO:teuthology.orchestra.run.smithi018.stderr:Creating mon... 2024-09-21T17:14:45.788 INFO:teuthology.orchestra.run.smithi018.stderr:create mon.smithi018 on 2024-09-21T17:14:46.012 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T17:14:46.172 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ec44ef70-783c-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-ec44ef70-783c-11ef-baf6-efdc52797490.target. 2024-09-21T17:14:46.172 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-ec44ef70-783c-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-ec44ef70-783c-11ef-baf6-efdc52797490.target. 2024-09-21T17:14:46.669 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Failed to reset failed state of unit ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018.service: Unit ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018.service not loaded. 2024-09-21T17:14:46.677 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ec44ef70-783c-11ef-baf6-efdc52797490.target.wants/ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018.service → /etc/systemd/system/ceph-ec44ef70-783c-11ef-baf6-efdc52797490@.service. 2024-09-21T17:14:47.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:47 smithi018 podman[23381]: 2024-09-21 17:14:47.083829389 +0000 UTC m=+0.117171390 container create 7fd171c51299d2d4ff243eb4b8ec11c00815cd7332cfc54017b79c08c62ed2e6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-21T17:14:47.384 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: disabled 2024-09-21T17:14:47.391 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-21T17:14:47.392 INFO:teuthology.orchestra.run.smithi018.stderr:firewalld.service is not enabled 2024-09-21T17:14:47.392 INFO:teuthology.orchestra.run.smithi018.stderr:Not possible to enable service . firewalld.service is not available 2024-09-21T17:14:47.392 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mon to start... 2024-09-21T17:14:47.392 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mon... 2024-09-21T17:14:47.517 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:47 smithi018 podman[23381]: 2024-09-21 17:14:47.29460656 +0000 UTC m=+0.327948564 container init 7fd171c51299d2d4ff243eb4b8ec11c00815cd7332cfc54017b79c08c62ed2e6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.openshift.tags=base centos centos-stream) 2024-09-21T17:14:47.517 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:47 smithi018 podman[23381]: 2024-09-21 17:14:47.298256935 +0000 UTC m=+0.331598936 container start 7fd171c51299d2d4ff243eb4b8ec11c00815cd7332cfc54017b79c08c62ed2e6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.openshift.expose-services=, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., name=centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux ) 2024-09-21T17:14:47.517 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:47 smithi018 ceph-mon[23400]: mkfs ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:47.517 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:47 smithi018 ceph-mon[23400]: mon.smithi018 is new leader, mons smithi018 in quorum (ranks 0) 2024-09-21T17:14:47.517 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:47 smithi018 bash[23381]: 7fd171c51299d2d4ff243eb4b8ec11c00815cd7332cfc54017b79c08c62ed2e6 2024-09-21T17:14:47.518 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:47 smithi018 systemd[1]: Started Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:14:47.955 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: cluster: 2024-09-21T17:14:47.955 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: id: ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: services: 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi018 (age 0.618049s) 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: data: 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-21T17:14:47.956 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-21T17:14:47.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: pgs: 2024-09-21T17:14:47.957 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:14:48.505 INFO:teuthology.orchestra.run.smithi018.stderr:mon is available 2024-09-21T17:14:48.505 INFO:teuthology.orchestra.run.smithi018.stderr:Assimilating anything we can from ceph.conf... 2024-09-21T17:14:48.628 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:48 smithi018 ceph-mon[23400]: mon.smithi018 is new leader, mons smithi018 in quorum (ranks 0) 2024-09-21T17:14:48.628 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:48 smithi018 ceph-mon[23400]: monmap e1: 1 mons at {smithi018=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0]} 2024-09-21T17:14:48.628 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:48 smithi018 ceph-mon[23400]: fsmap 2024-09-21T17:14:48.628 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:48 smithi018 ceph-mon[23400]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T17:14:48.628 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:48 smithi018 ceph-mon[23400]: mgrmap e1: no daemons active 2024-09-21T17:14:48.628 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:48 smithi018 ceph-mon[23400]: from='client.? 172.21.15.18:0/3305911886' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T17:14:49.019 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:14:49.019 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [global] 2024-09-21T17:14:49.019 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: fsid = ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:49.019 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.18:3300,v1:172.21.15.18:6789] 2024-09-21T17:14:49.019 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-21T17:14:49.019 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [mgr] 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [osd] 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-21T17:14:49.020 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-21T17:14:49.592 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:49 smithi018 ceph-mon[23400]: from='client.? 172.21.15.18:0/2696457622' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T17:14:49.592 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:49 smithi018 ceph-mon[23400]: from='client.? 172.21.15.18:0/2696457622' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T17:14:49.594 INFO:teuthology.orchestra.run.smithi018.stderr:Generating new minimal ceph.conf... 2024-09-21T17:14:50.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:50 smithi018 ceph-mon[23400]: from='client.? 172.21.15.18:0/1884507217' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:14:50.675 INFO:teuthology.orchestra.run.smithi018.stderr:Restarting the monitor... 2024-09-21T17:14:51.126 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:50 smithi018 systemd[1]: Stopping Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:14:51.126 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:50 smithi018 bash[23618]: Error: no container with name or ID "ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon.smithi018" found: no such container 2024-09-21T17:14:51.126 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:50 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[23396]: 2024-09-21T17:14:50.823+0000 7fc7e1bdd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi018 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:14:51.126 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:50 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[23396]: 2024-09-21T17:14:50.823+0000 7fc7e1bdd700 -1 mon.smithi018@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T17:14:51.126 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:50 smithi018 podman[23629]: 2024-09-21 17:14:50.866276277 +0000 UTC m=+0.139010817 container died 7fd171c51299d2d4ff243eb4b8ec11c00815cd7332cfc54017b79c08c62ed2e6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., name=centos-stream, release=754, architecture=x86_64, io.openshift.expose-services=) 2024-09-21T17:14:51.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 podman[23629]: 2024-09-21 17:14:51.123968718 +0000 UTC m=+0.396703260 container cleanup 7fd171c51299d2d4ff243eb4b8ec11c00815cd7332cfc54017b79c08c62ed2e6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, ceph=True, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.buildah.version=1.19.8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , name=centos-stream, distribution-scope=public, architecture=x86_64) 2024-09-21T17:14:51.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 bash[23629]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018 2024-09-21T17:14:51.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 bash[23656]: Error: no container with name or ID "ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon.smithi018" found: no such container 2024-09-21T17:14:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 podman[23641]: 2024-09-21 17:14:51.375054592 +0000 UTC m=+0.510766200 container remove 7fd171c51299d2d4ff243eb4b8ec11c00815cd7332cfc54017b79c08c62ed2e6 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.openshift.expose-services=, architecture=x86_64) 2024-09-21T17:14:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018.service: Deactivated successfully. 2024-09-21T17:14:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 systemd[1]: Stopped Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:14:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 systemd[1]: Starting Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:14:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 podman[23719]: 2024-09-21 17:14:51.600705096 +0000 UTC m=+0.100566162 container create 405cef9a3f40cbe8e0248758cccde1cc7f6cdd2ad26795d083c1090fdd697926 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, GIT_BRANCH=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_CLEAN=True, io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, release=754) 2024-09-21T17:14:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 podman[23719]: 2024-09-21 17:14:51.517620394 +0000 UTC m=+0.017481472 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-21T17:14:51.874 INFO:teuthology.orchestra.run.smithi018.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-21T17:14:51.980 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 podman[23719]: 2024-09-21 17:14:51.794927086 +0000 UTC m=+0.294788163 container init 405cef9a3f40cbe8e0248758cccde1cc7f6cdd2ad26795d083c1090fdd697926 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-type=git, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, ceph=True, RELEASE=HEAD, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container) 2024-09-21T17:14:51.980 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 podman[23719]: 2024-09-21 17:14:51.798345266 +0000 UTC m=+0.298206334 container start 405cef9a3f40cbe8e0248758cccde1cc7f6cdd2ad26795d083c1090fdd697926 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, io.openshift.expose-services=, release=754, ceph=True, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, version=8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, vendor=Red Hat, Inc., vcs-type=git, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-09-21T17:14:51.980 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T17:14:51.980 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-21T17:14:51.980 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: pidfile_write: ignore empty --pid-file 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: load: jerasure load: lrc 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: RocksDB version: 6.15.5 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Compile date Apr 18 2022 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: DB SUMMARY 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: DB Session ID: I4DA7NSO9WSYISUISOM8 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: CURRENT file: CURRENT 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-21T17:14:51.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi018/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi018/store.db: 000010.log size: 82715 ; 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.error_if_exists: 0 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.create_if_missing: 0 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.paranoid_checks: 1 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.env: 0x55774dec6860 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.fs: Posix File System 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.info_log: 0x5577506b3fa0 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.statistics: (nil) 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.use_fsync: 0 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_log_file_size: 0 2024-09-21T17:14:51.982 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.allow_fallocate: 1 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.use_direct_reads: 0 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.db_log_dir: 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi018/store.db 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T17:14:51.983 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T17:14:51.984 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T17:14:51.984 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T17:14:51.984 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T17:14:51.984 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.write_buffer_manager: 0x5577507a4330 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T17:14:51.985 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.unordered_write: 0 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.row_cache: None 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.wal_filter: None 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T17:14:51.986 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.preserve_deletes: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.two_write_queues: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.atomic_flush: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.log_readahead_size: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_background_jobs: 2 2024-09-21T17:14:51.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_background_compactions: -1 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_subcompactions: 1 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_open_files: -1 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T17:14:51.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_background_flushes: -1 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Compression algorithms supported: 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: kZSTD supported: 0 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: kXpressCompression supported: 0 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: kLZ4Compression supported: 1 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: kBZip2Compression supported: 0 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: kZlibCompression supported: 1 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: kSnappyCompression supported: 1 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi018/store.db/MANIFEST-000009 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.merge_operator: 2024-09-21T17:14:51.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_filter: None 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55775067fd00) 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: cache_index_and_filter_blocks: 1 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: pin_top_level_index_and_filter: 1 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: index_type: 0 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: data_block_index_type: 0 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: index_shortening: 1 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T17:14:51.990 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: hash_index_allow_collision: 1 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: checksum: 1 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: no_block_cache: 0 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_cache: 0x5577506ea170 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_cache_name: BinnedLRUCache 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_cache_options: 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: capacity : 536870912 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: num_shard_bits : 4 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: strict_capacity_limit : 0 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: high_pri_pool_ratio: 0.000 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_cache_compressed: (nil) 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: persistent_cache: (nil) 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_size: 4096 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_size_deviation: 10 2024-09-21T17:14:51.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_restart_interval: 16 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: index_block_restart_interval: 1 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: metadata_block_size: 4096 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: partition_filters: 0 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: use_delta_encoding: 1 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: whole_key_filtering: 1 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: verify_compression: 0 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: read_amp_bytes_per_bit: 0 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: format_version: 4 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: enable_index_compression: 1 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_align: 0 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T17:14:51.992 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compression: NoCompression 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.num_levels: 7 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T17:14:51.993 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T17:14:51.994 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T17:14:51.995 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.table_properties_collectors: 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.inplace_update_support: 0 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T17:14:51.996 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.bloom_locality: 0 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.max_successive_merges: 0 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.ttl: 2592000 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.enable_blob_files: false 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.min_blob_size: 0 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T17:14:51.997 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi018/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726938891825272, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726938891826069, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79397, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77661, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71970, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726938891, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "822a755b-4721-463f-bf1c-76daf2f8bed8", "db_session_id": "I4DA7NSO9WSYISUISOM8"}} 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726938891826399, "job": 1, "event": "recovery_finished"} 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi018/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5577506d0a80 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: DB pointer 0x5577506e0000 2024-09-21T17:14:51.998 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** DB Stats ** 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.9 0.00 0.00 1 0.001 0 0 2024-09-21T17:14:51.999 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.9 0.00 0.00 1 0.001 0 0 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.9 0.00 0.00 1 0.001 0 0 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 142.9 0.00 0.00 1 0.001 0 0 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative compaction: 0.00 GB write, 24.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T17:14:52.000 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval compaction: 0.00 GB write, 24.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.9 0.00 0.00 1 0.001 0 0 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.9 0.00 0.00 1 0.001 0 0 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T17:14:52.001 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 142.9 0.00 0.00 1 0.001 0 0 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative compaction: 0.00 GB write, 24.04 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: starting mon.smithi018 rank 0 at public addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] at bind addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi018 fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:52.002 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018@-1(???) e1 preinit fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018@-1(???).mds e1 new map 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018@-1(???).mds e1 print_map 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: e1 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: legacy client fscid: -1 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: No filesystems configured 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta expand map: {default=false} 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta from 'false' to 'false' 2024-09-21T17:14:52.003 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta expanded map: {default=false} 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta expand map: {default=info} 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta from 'info' to 'info' 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta expanded map: {default=info} 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta expand map: {default=daemon} 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta expanded map: {default=daemon} 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta expand map: {default=debug} 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta from 'debug' to 'debug' 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: expand_channel_meta expanded map: {default=debug} 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 bash[23719]: 405cef9a3f40cbe8e0248758cccde1cc7f6cdd2ad26795d083c1090fdd697926 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 systemd[1]: Started Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mon.smithi018 is new leader, mons smithi018 in quorum (ranks 0) 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: monmap e1: 1 mons at {smithi018=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0]} 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: fsmap 2024-09-21T17:14:52.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T17:14:52.005 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:51 smithi018 ceph-mon[23738]: mgrmap e1: no daemons active 2024-09-21T17:14:52.989 INFO:teuthology.orchestra.run.smithi018.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-21T17:14:52.989 INFO:teuthology.orchestra.run.smithi018.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T17:14:52.990 INFO:teuthology.orchestra.run.smithi018.stderr:Creating mgr... 2024-09-21T17:14:52.990 INFO:teuthology.orchestra.run.smithi018.stderr:Verifying port 9283 ... 2024-09-21T17:14:53.178 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Failed to reset failed state of unit ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mgr.smithi018.jndyyn.service: Unit ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mgr.smithi018.jndyyn.service not loaded. 2024-09-21T17:14:53.186 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ec44ef70-783c-11ef-baf6-efdc52797490.target.wants/ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mgr.smithi018.jndyyn.service → /etc/systemd/system/ceph-ec44ef70-783c-11ef-baf6-efdc52797490@.service. 2024-09-21T17:14:53.731 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:53 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/4162409780' entity='client.admin' 2024-09-21T17:14:53.956 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: disabled 2024-09-21T17:14:53.964 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-21T17:14:53.965 INFO:teuthology.orchestra.run.smithi018.stderr:firewalld.service is not enabled 2024-09-21T17:14:53.965 INFO:teuthology.orchestra.run.smithi018.stderr:Not possible to enable service . firewalld.service is not available 2024-09-21T17:14:53.975 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: disabled 2024-09-21T17:14:53.984 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-21T17:14:53.985 INFO:teuthology.orchestra.run.smithi018.stderr:firewalld.service is not enabled 2024-09-21T17:14:53.985 INFO:teuthology.orchestra.run.smithi018.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-21T17:14:53.985 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mgr to start... 2024-09-21T17:14:53.985 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mgr... 2024-09-21T17:14:54.981 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsid": "ec44ef70-783c-11ef-baf6-efdc52797490", 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "health": { 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 0 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:14:54.982 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "smithi018" 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T17:14:54.983 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T17:14:54.987 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": false, 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "iostat", 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "nfs", 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "restful" 2024-09-21T17:14:54.988 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modified": "2024-09-21T17:14:47.334821+0000", 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T17:14:54.989 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-21T17:14:55.238 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:55 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1537950767' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T17:14:55.791 INFO:teuthology.orchestra.run.smithi018.stderr:mgr not available, waiting (1/15)... 2024-09-21T17:14:58.318 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:14:58.318 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-21T17:14:58.318 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsid": "ec44ef70-783c-11ef-baf6-efdc52797490", 2024-09-21T17:14:58.318 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "health": { 2024-09-21T17:14:58.318 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T17:14:58.318 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 0 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "smithi018" 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:14:58.319 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T17:14:58.320 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T17:14:58.320 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:58.320 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T17:14:58.320 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:14:58.320 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T17:14:58.320 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T17:14:58.320 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T17:14:58.320 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T17:14:58.323 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T17:14:58.323 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T17:14:58.323 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:58.323 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:58.324 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": false, 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "iostat", 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "nfs", 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "restful" 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modified": "2024-09-21T17:14:47.334821+0000", 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:14:58.325 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T17:14:58.326 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-21T17:14:58.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:14:58 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/535121977' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T17:14:58.965 INFO:teuthology.orchestra.run.smithi018.stderr:mgr not available, waiting (2/15)... 2024-09-21T17:15:00.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: Activating manager daemon smithi018.jndyyn 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: mgrmap e2: smithi018.jndyyn(active, starting, since 0.00282889s) 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: Manager daemon smithi018.jndyyn is now available 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:00 smithi018 ceph-mon[23738]: from='mgr.14100 172.21.15.18:0/3213931150' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:01.676 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsid": "ec44ef70-783c-11ef-baf6-efdc52797490", 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "health": { 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 0 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "smithi018" 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:15:01.677 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T17:15:01.678 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T17:15:01.681 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T17:15:01.681 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T17:15:01.681 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:15:01.681 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:15:01.682 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": true, 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "iostat", 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "nfs", 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "restful" 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ], 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "modified": "2024-09-21T17:14:47.334821+0000", 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "services": {} 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: }, 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T17:15:01.683 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-21T17:15:02.284 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:02 smithi018 ceph-mon[23738]: mgrmap e3: smithi018.jndyyn(active, since 1.00594s) 2024-09-21T17:15:02.284 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:02 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1210855950' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T17:15:02.287 INFO:teuthology.orchestra.run.smithi018.stderr:mgr is available 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [global] 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: fsid = ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:15:02.902 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [mgr] 2024-09-21T17:15:02.903 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-21T17:15:02.903 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 2024-09-21T17:15:02.903 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: [osd] 2024-09-21T17:15:02.903 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-21T17:15:02.903 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-21T17:15:03.460 INFO:teuthology.orchestra.run.smithi018.stderr:Enabling cephadm module... 2024-09-21T17:15:04.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:03 smithi018 ceph-mon[23738]: mgrmap e4: smithi018.jndyyn(active, since 2s) 2024-09-21T17:15:04.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:03 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/537660419' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T17:15:05.098 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:04 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/355563428' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T17:15:06.086 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:05 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/355563428' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T17:15:06.087 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:05 smithi018 ceph-mon[23738]: mgrmap e5: smithi018.jndyyn(active, since 4s) 2024-09-21T17:15:06.132 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-21T17:15:06.132 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-21T17:15:06.132 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": true, 2024-09-21T17:15:06.132 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "active_name": "smithi018.jndyyn", 2024-09-21T17:15:06.132 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-21T17:15:06.132 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-21T17:15:06.748 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for the mgr to restart... 2024-09-21T17:15:06.748 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mgr epoch 5... 2024-09-21T17:15:07.055 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:06 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1795080810' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T17:15:12.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: Active manager daemon smithi018.jndyyn restarted 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: Activating manager daemon smithi018.jndyyn 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: mgrmap e6: smithi018.jndyyn(active, starting, since 0.0523683s) 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: Manager daemon smithi018.jndyyn is now available 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:15:12.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:15:12.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:15:12.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:12 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:15:12.243 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-21T17:15:12.243 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-21T17:15:12.243 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "initialized": true 2024-09-21T17:15:12.243 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-21T17:15:12.837 INFO:teuthology.orchestra.run.smithi018.stderr:mgr epoch 5 is available 2024-09-21T17:15:12.837 INFO:teuthology.orchestra.run.smithi018.stderr:Setting orchestrator backend to cephadm... 2024-09-21T17:15:13.512 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:13 smithi018 ceph-mon[23738]: [21/Sep/2024:17:15:12] ENGINE Bus STARTING 2024-09-21T17:15:13.512 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:13 smithi018 ceph-mon[23738]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T17:15:13.512 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:13 smithi018 ceph-mon[23738]: mgrmap e7: smithi018.jndyyn(active, since 1.05533s) 2024-09-21T17:15:13.512 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:13 smithi018 ceph-mon[23738]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T17:15:13.512 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:13 smithi018 ceph-mon[23738]: [21/Sep/2024:17:15:12] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T17:15:13.512 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:13 smithi018 ceph-mon[23738]: [21/Sep/2024:17:15:12] ENGINE Bus STARTED 2024-09-21T17:15:13.512 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:13 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:13.512 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:13 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:15:14.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:14 smithi018 ceph-mon[23738]: mgrmap e8: smithi018.jndyyn(active, since 2s) 2024-09-21T17:15:14.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:14 smithi018 ceph-mon[23738]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:14.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:14 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:14.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:14 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:15:14.770 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: value unchanged 2024-09-21T17:15:15.525 INFO:teuthology.orchestra.run.smithi018.stderr:Generating ssh key... 2024-09-21T17:15:15.651 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:15 smithi018 ceph-mon[23738]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:17.973 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:17 smithi018 ceph-mon[23738]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:17.973 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:17 smithi018 ceph-mon[23738]: Generating ssh key... 2024-09-21T17:15:17.973 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:17 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:17.973 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:17 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:17.973 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ0GoH926F9KOrh983amEniNpdrKeKjHVmLWnZYD5FUW94s5gNYbyetG2r2bIFpsH0sURLerdB+WEx6uOpN3hSKr2ELjYl3sejgQL9AGDQTorwlSzgTTA3OUJ9zWG7o+njGTxjpBNmrMHJboRqab0ZzAWK8rGhyVZxF6XotpIstbb596NxtmKIKR+dmAy8vksYlD80xNZIp1Hnmb1xchPktJ0kLKK43nf95IjXqEYc1a4D0MwgG57WB3ueJ7ZVIRh/kpBwF2KHf/SRL46gmOJhIjeZALX0FZd4wWk1aQiTWWaY0uS2FxjEfKFeZeKvny6cBrhvogP0PXj491e1hYf8q1SWiPOEAPmeDSfYqwafG9bV2Kr6LHYJ32BDC+ojW5usgNi9/JBIU/Ye7NYa42L7UKMH2BFqO7wa0pNQfOlimJvtD0Ni7VKcrxW1Di5L1uyy9c1xXBGHQfw5sdLJYxd44Edb9ciQAPj1uyOWVmUzOCVgyjJSaYQ5ZF3poQyPdWM= ceph-ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:15:18.562 INFO:teuthology.orchestra.run.smithi018.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T17:15:18.562 INFO:teuthology.orchestra.run.smithi018.stderr:Adding key to root@localhost authorized_keys... 2024-09-21T17:15:18.570 INFO:teuthology.orchestra.run.smithi018.stderr:Adding host smithi018... 2024-09-21T17:15:19.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:18 smithi018 ceph-mon[23738]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:20.112 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Added host 'smithi018' with addr '172.21.15.18' 2024-09-21T17:15:20.124 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:19 smithi018 ceph-mon[23738]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi018", "addr": "172.21.15.18", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:20.124 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:19 smithi018 ceph-mon[23738]: Deploying cephadm binary to smithi018 2024-09-21T17:15:20.711 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying mon service with default placement... 2024-09-21T17:15:21.364 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:21 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:21.364 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:21 smithi018 ceph-mon[23738]: Added host smithi018 2024-09-21T17:15:21.364 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:21 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:15:21.365 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-21T17:15:22.154 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying mgr service with default placement... 2024-09-21T17:15:22.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:22 smithi018 ceph-mon[23738]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:22.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:22 smithi018 ceph-mon[23738]: Saving service mon spec with placement count:5 2024-09-21T17:15:22.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:22 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:22.924 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-21T17:15:24.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:23 smithi018 ceph-mon[23738]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:24.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:23 smithi018 ceph-mon[23738]: Saving service mgr spec with placement count:2 2024-09-21T17:15:24.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:23 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:24.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:23 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:24.231 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying crash service with default placement... 2024-09-21T17:15:24.976 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-21T17:15:25.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:25 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:25 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:26.034 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying prometheus service with default placement... 2024-09-21T17:15:26.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:26 smithi018 ceph-mon[23738]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:26.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:26 smithi018 ceph-mon[23738]: Saving service crash spec with placement * 2024-09-21T17:15:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:26 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:26.712 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-21T17:15:27.365 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying grafana service with default placement... 2024-09-21T17:15:27.925 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:27 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:27.925 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:27 smithi018 ceph-mon[23738]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:27.925 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:27 smithi018 ceph-mon[23738]: Saving service prometheus spec with placement count:1 2024-09-21T17:15:27.925 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:27 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:27.978 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-21T17:15:28.660 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying node-exporter service with default placement... 2024-09-21T17:15:29.237 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:28 smithi018 ceph-mon[23738]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:29.237 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:28 smithi018 ceph-mon[23738]: Saving service grafana spec with placement count:1 2024-09-21T17:15:29.237 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:28 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:29.306 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-21T17:15:30.350 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:30 smithi018 ceph-mon[23738]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:30.350 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:30 smithi018 ceph-mon[23738]: Saving service node-exporter spec with placement * 2024-09-21T17:15:30.351 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:30 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:30.353 INFO:teuthology.orchestra.run.smithi018.stderr:Deploying alertmanager service with default placement... 2024-09-21T17:15:31.097 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-21T17:15:32.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:32 smithi018 ceph-mon[23738]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:32.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:32 smithi018 ceph-mon[23738]: Saving service alertmanager spec with placement count:1 2024-09-21T17:15:32.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:32 smithi018 ceph-mon[23738]: from='mgr.14118 172.21.15.18:0/3624852750' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:32.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:32 smithi018 ceph-mon[23738]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:15:33.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:33 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/137309889' entity='client.admin' 2024-09-21T17:15:34.388 INFO:teuthology.orchestra.run.smithi018.stderr:Enabling the dashboard module... 2024-09-21T17:15:34.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:34 smithi018 ceph-mon[23738]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:15:34.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:34 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1785029349' entity='client.admin' 2024-09-21T17:15:35.951 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:35 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/391618575' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T17:15:36.848 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:36 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/391618575' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T17:15:36.944 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:36 smithi018 ceph-mon[23738]: mgrmap e9: smithi018.jndyyn(active, since 24s) 2024-09-21T17:15:37.078 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-21T17:15:37.079 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-21T17:15:37.079 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "available": true, 2024-09-21T17:15:37.079 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "active_name": "smithi018.jndyyn", 2024-09-21T17:15:37.079 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-21T17:15:37.079 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-21T17:15:37.710 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for the mgr to restart... 2024-09-21T17:15:37.710 INFO:teuthology.orchestra.run.smithi018.stderr:Waiting for mgr epoch 9... 2024-09-21T17:15:37.973 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:37 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2074550564' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: Active manager daemon smithi018.jndyyn restarted 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: Activating manager daemon smithi018.jndyyn 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: mgrmap e10: smithi018.jndyyn(active, starting, since 0.761047s) 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T17:15:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: Manager daemon smithi018.jndyyn is now available 2024-09-21T17:15:43.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:15:43.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:15:43.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:15:43.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:43 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:15:43.852 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: { 2024-09-21T17:15:43.852 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-21T17:15:43.852 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: "initialized": true 2024-09-21T17:15:43.852 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: } 2024-09-21T17:15:44.560 INFO:teuthology.orchestra.run.smithi018.stderr:mgr epoch 9 is available 2024-09-21T17:15:44.560 INFO:teuthology.orchestra.run.smithi018.stderr:Generating a dashboard self-signed certificate... 2024-09-21T17:15:45.087 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:44 smithi018 ceph-mon[23738]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T17:15:45.087 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:44 smithi018 ceph-mon[23738]: mgrmap e11: smithi018.jndyyn(active, since 1.76345s) 2024-09-21T17:15:45.087 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:44 smithi018 ceph-mon[23738]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T17:15:45.087 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:44 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:45.376 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-21T17:15:46.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: mgrmap e12: smithi018.jndyyn(active, since 2s) 2024-09-21T17:15:46.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: [21/Sep/2024:17:15:44] ENGINE Bus STARTING 2024-09-21T17:15:46.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:46.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: [21/Sep/2024:17:15:45] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T17:15:46.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: [21/Sep/2024:17:15:45] ENGINE Bus STARTED 2024-09-21T17:15:46.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:46.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:46.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:46.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:45 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:46.505 INFO:teuthology.orchestra.run.smithi018.stderr:Creating initial admin user... 2024-09-21T17:15:47.430 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$SARIajTTn1rhX.D9mVuVx.6tdewZt.IRZNOxEXVWHda8f3hY9kM12", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726938947, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T17:15:48.062 INFO:teuthology.orchestra.run.smithi018.stderr:Fetching dashboard port number... 2024-09-21T17:15:48.786 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:48 smithi018 ceph-mon[23738]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:15:48.786 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:48 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:48.786 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: 8443 2024-09-21T17:15:49.440 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: disabled 2024-09-21T17:15:49.449 INFO:teuthology.orchestra.run.smithi018.stderr:systemctl: inactive 2024-09-21T17:15:49.450 INFO:teuthology.orchestra.run.smithi018.stderr:firewalld.service is not enabled 2024-09-21T17:15:49.450 INFO:teuthology.orchestra.run.smithi018.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T17:15:49.453 INFO:teuthology.orchestra.run.smithi018.stderr:Ceph Dashboard is now available at: 2024-09-21T17:15:49.453 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:15:49.453 INFO:teuthology.orchestra.run.smithi018.stderr: URL: https://smithi018.front.sepia.ceph.com:8443/ 2024-09-21T17:15:49.453 INFO:teuthology.orchestra.run.smithi018.stderr: User: admin 2024-09-21T17:15:49.453 INFO:teuthology.orchestra.run.smithi018.stderr: Password: ssi52mf9vs 2024-09-21T17:15:49.453 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:15:49.453 INFO:teuthology.orchestra.run.smithi018.stderr:Enabling autotune for osd_memory_target 2024-09-21T17:15:49.707 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:49 smithi018 ceph-mon[23738]: mgrmap e13: smithi018.jndyyn(active, since 6s) 2024-09-21T17:15:49.707 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:49 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3035134802' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T17:15:51.494 INFO:teuthology.orchestra.run.smithi018.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr:You can access the Ceph CLI with: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: ceph telemetry on 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr:For more information see: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:15:52.108 INFO:teuthology.orchestra.run.smithi018.stderr:Bootstrap complete. 2024-09-21T17:15:52.147 INFO:tasks.cephadm:Fetching config... 2024-09-21T17:15:52.147 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:15:52.148 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T17:15:52.165 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T17:15:52.165 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:15:52.166 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T17:15:52.222 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T17:15:52.222 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:15:52.223 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/keyring of=/dev/stdout 2024-09-21T17:15:52.290 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T17:15:52.290 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:15:52.290 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T17:15:52.346 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T17:15:52.347 DEBUG:teuthology.orchestra.run.smithi018:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ0GoH926F9KOrh983amEniNpdrKeKjHVmLWnZYD5FUW94s5gNYbyetG2r2bIFpsH0sURLerdB+WEx6uOpN3hSKr2ELjYl3sejgQL9AGDQTorwlSzgTTA3OUJ9zWG7o+njGTxjpBNmrMHJboRqab0ZzAWK8rGhyVZxF6XotpIstbb596NxtmKIKR+dmAy8vksYlD80xNZIp1Hnmb1xchPktJ0kLKK43nf95IjXqEYc1a4D0MwgG57WB3ueJ7ZVIRh/kpBwF2KHf/SRL46gmOJhIjeZALX0FZd4wWk1aQiTWWaY0uS2FxjEfKFeZeKvny6cBrhvogP0PXj491e1hYf8q1SWiPOEAPmeDSfYqwafG9bV2Kr6LHYJ32BDC+ojW5usgNi9/JBIU/Ye7NYa42L7UKMH2BFqO7wa0pNQfOlimJvtD0Ni7VKcrxW1Di5L1uyy9c1xXBGHQfw5sdLJYxd44Edb9ciQAPj1uyOWVmUzOCVgyjJSaYQ5ZF3poQyPdWM= ceph-ec44ef70-783c-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T17:15:52.423 INFO:teuthology.orchestra.run.smithi018.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ0GoH926F9KOrh983amEniNpdrKeKjHVmLWnZYD5FUW94s5gNYbyetG2r2bIFpsH0sURLerdB+WEx6uOpN3hSKr2ELjYl3sejgQL9AGDQTorwlSzgTTA3OUJ9zWG7o+njGTxjpBNmrMHJboRqab0ZzAWK8rGhyVZxF6XotpIstbb596NxtmKIKR+dmAy8vksYlD80xNZIp1Hnmb1xchPktJ0kLKK43nf95IjXqEYc1a4D0MwgG57WB3ueJ7ZVIRh/kpBwF2KHf/SRL46gmOJhIjeZALX0FZd4wWk1aQiTWWaY0uS2FxjEfKFeZeKvny6cBrhvogP0PXj491e1hYf8q1SWiPOEAPmeDSfYqwafG9bV2Kr6LHYJ32BDC+ojW5usgNi9/JBIU/Ye7NYa42L7UKMH2BFqO7wa0pNQfOlimJvtD0Ni7VKcrxW1Di5L1uyy9c1xXBGHQfw5sdLJYxd44Edb9ciQAPj1uyOWVmUzOCVgyjJSaYQ5ZF3poQyPdWM= ceph-ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:15:52.434 DEBUG:teuthology.orchestra.run.smithi157:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ0GoH926F9KOrh983amEniNpdrKeKjHVmLWnZYD5FUW94s5gNYbyetG2r2bIFpsH0sURLerdB+WEx6uOpN3hSKr2ELjYl3sejgQL9AGDQTorwlSzgTTA3OUJ9zWG7o+njGTxjpBNmrMHJboRqab0ZzAWK8rGhyVZxF6XotpIstbb596NxtmKIKR+dmAy8vksYlD80xNZIp1Hnmb1xchPktJ0kLKK43nf95IjXqEYc1a4D0MwgG57WB3ueJ7ZVIRh/kpBwF2KHf/SRL46gmOJhIjeZALX0FZd4wWk1aQiTWWaY0uS2FxjEfKFeZeKvny6cBrhvogP0PXj491e1hYf8q1SWiPOEAPmeDSfYqwafG9bV2Kr6LHYJ32BDC+ojW5usgNi9/JBIU/Ye7NYa42L7UKMH2BFqO7wa0pNQfOlimJvtD0Ni7VKcrxW1Di5L1uyy9c1xXBGHQfw5sdLJYxd44Edb9ciQAPj1uyOWVmUzOCVgyjJSaYQ5ZF3poQyPdWM= ceph-ec44ef70-783c-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T17:15:52.625 INFO:teuthology.orchestra.run.smithi157.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCZ0GoH926F9KOrh983amEniNpdrKeKjHVmLWnZYD5FUW94s5gNYbyetG2r2bIFpsH0sURLerdB+WEx6uOpN3hSKr2ELjYl3sejgQL9AGDQTorwlSzgTTA3OUJ9zWG7o+njGTxjpBNmrMHJboRqab0ZzAWK8rGhyVZxF6XotpIstbb596NxtmKIKR+dmAy8vksYlD80xNZIp1Hnmb1xchPktJ0kLKK43nf95IjXqEYc1a4D0MwgG57WB3ueJ7ZVIRh/kpBwF2KHf/SRL46gmOJhIjeZALX0FZd4wWk1aQiTWWaY0uS2FxjEfKFeZeKvny6cBrhvogP0PXj491e1hYf8q1SWiPOEAPmeDSfYqwafG9bV2Kr6LHYJ32BDC+ojW5usgNi9/JBIU/Ye7NYa42L7UKMH2BFqO7wa0pNQfOlimJvtD0Ni7VKcrxW1Di5L1uyy9c1xXBGHQfw5sdLJYxd44Edb9ciQAPj1uyOWVmUzOCVgyjJSaYQ5ZF3poQyPdWM= ceph-ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:15:52.639 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T17:15:52.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:52 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1513972311' entity='client.admin' 2024-09-21T17:15:56.785 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T17:15:56.785 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T17:15:57.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:56 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2782034981' entity='client.admin' 2024-09-21T17:15:58.286 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:58 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:58.286 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:58 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd/host:smithi018", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:15:58.286 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:58 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:15:58.286 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:15:58 smithi018 ceph-mon[23738]: Deploying daemon alertmanager.smithi018 on smithi018 2024-09-21T17:15:59.976 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi157 2024-09-21T17:15:59.976 DEBUG:teuthology.orchestra.run.smithi157:> set -ex 2024-09-21T17:15:59.976 DEBUG:teuthology.orchestra.run.smithi157:> dd of=/etc/ceph/ceph.conf 2024-09-21T17:15:59.995 DEBUG:teuthology.orchestra.run.smithi157:> set -ex 2024-09-21T17:15:59.995 DEBUG:teuthology.orchestra.run.smithi157:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:16:00.053 INFO:tasks.cephadm:Adding host smithi157 to orchestrator... 2024-09-21T17:16:00.053 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch host add smithi157 2024-09-21T17:16:00.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:00 smithi018 ceph-mon[23738]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:16:00.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:00 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:04.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:03 smithi018 ceph-mon[23738]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:04.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:03 smithi018 ceph-mon[23738]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi157", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:16:04.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:03 smithi018 ceph-mon[23738]: Deploying cephadm binary to smithi157 2024-09-21T17:16:04.219 INFO:teuthology.orchestra.run.smithi018.stdout:Added host 'smithi157' with addr '172.21.15.157' 2024-09-21T17:16:05.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:05 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:05.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:05 smithi018 ceph-mon[23738]: Added host smithi157 2024-09-21T17:16:05.888 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T17:16:06.626 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:06 smithi018 ceph-mon[23738]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:07.976 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:07 smithi018 ceph-mon[23738]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:08.412 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:16:08.412 INFO:teuthology.orchestra.run.smithi018.stdout:[{"addr": "172.21.15.18", "hostname": "smithi018", "labels": [], "status": ""}, {"addr": "172.21.15.157", "hostname": "smithi157", "labels": [], "status": ""}] 2024-09-21T17:16:08.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:08 smithi018 ceph-mon[23738]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:16:09.603 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T17:16:09.603 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd crush tunables default 2024-09-21T17:16:10.090 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:09 smithi018 ceph-mon[23738]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:10.091 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:09 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:10.091 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:09 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi018", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T17:16:10.091 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:09 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi018", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T17:16:10.091 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:09 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:16:10.091 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:09 smithi018 ceph-mon[23738]: Deploying daemon crash.smithi018 on smithi018 2024-09-21T17:16:12.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:11 smithi018 ceph-mon[23738]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:12.998 INFO:teuthology.orchestra.run.smithi018.stderr:adjusted tunables profile to default 2024-09-21T17:16:13.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:12 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3253780228' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T17:16:13.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:12 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:13.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:12 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:13.645 INFO:tasks.cephadm:Adding mon.smithi018 on smithi018 2024-09-21T17:16:13.645 INFO:tasks.cephadm:Adding mon.smithi157 on smithi157 2024-09-21T17:16:13.645 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch apply mon '2;smithi018:172.21.15.18=smithi018;smithi157:172.21.15.157=smithi157' 2024-09-21T17:16:14.405 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:14.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3253780228' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T17:16:14.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T17:16:14.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:14.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:14.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T17:16:14.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T17:16:14.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:14.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:13 smithi018 ceph-mon[23738]: Deploying daemon grafana.smithi018 on smithi018 2024-09-21T17:16:15.594 INFO:teuthology.orchestra.run.smithi157.stdout:Scheduled mon update... 2024-09-21T17:16:16.188 DEBUG:teuthology.orchestra.run.smithi157:mon.smithi157> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi157.service 2024-09-21T17:16:16.190 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:16.190 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:16.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:16 smithi018 ceph-mon[23738]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:16.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:16 smithi018 ceph-mon[23738]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi018:172.21.15.18=smithi018;smithi157:172.21.15.157=smithi157", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:16:16.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:16 smithi018 ceph-mon[23738]: Saving service mon spec with placement smithi018:172.21.15.18=smithi018;smithi157:172.21.15.157=smithi157;count:2 2024-09-21T17:16:16.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:16 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:18.115 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:18.115 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:18.117 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:18.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:17 smithi018 ceph-mon[23738]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:19.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:18 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/3492772576' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:19.757 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:19.757 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:20.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:19 smithi018 ceph-mon[23738]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:21.698 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:21.698 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:21.699 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:21.891 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:21 smithi018 ceph-mon[23738]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:21.891 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:21 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/3554678128' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:23.277 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:23.277 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:25.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:24 smithi018 ceph-mon[23738]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:25.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:24 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:25.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:24 smithi018 ceph-mon[23738]: Deploying daemon node-exporter.smithi018 on smithi018 2024-09-21T17:16:25.221 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:25.221 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:25.223 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:25.962 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:25 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/2179117188' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:26.830 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:26.831 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:26.954 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:26 smithi018 ceph-mon[23738]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:28.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:28 smithi018 ceph-mon[23738]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:28.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:28 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:28.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:28 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T17:16:28.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:28 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:28.706 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:28.706 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:28.708 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:29.361 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:29 smithi018 ceph-mon[23738]: from='mgr.14160 172.21.15.18:0/1648439304' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T17:16:29.361 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:29 smithi018 ceph-mon[23738]: mgrmap e14: smithi018.jndyyn(active, since 46s) 2024-09-21T17:16:29.361 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:29 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/2672216875' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:30.326 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:30.326 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:32.208 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:32.209 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:32.210 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:33.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:32 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/1259227604' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:33.819 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:33.819 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: Active manager daemon smithi018.jndyyn restarted 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: Activating manager daemon smithi018.jndyyn 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: mgrmap e15: smithi018.jndyyn(active, starting, since 0.0522464s) 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T17:16:35.606 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: Manager daemon smithi018.jndyyn is now available 2024-09-21T17:16:35.607 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:16:35.607 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:16:35.607 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:16:35.607 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:16:35.607 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:35.607 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:16:35.607 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:16:35.607 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:35.925 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:35.925 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:35.927 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:36.823 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:36 smithi018 ceph-mon[23738]: [21/Sep/2024:17:16:35] ENGINE Bus STARTING 2024-09-21T17:16:36.823 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:36 smithi018 ceph-mon[23738]: mgrmap e16: smithi018.jndyyn(active, since 1.05543s) 2024-09-21T17:16:36.823 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:36 smithi018 ceph-mon[23738]: [21/Sep/2024:17:16:35] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T17:16:36.824 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:36 smithi018 ceph-mon[23738]: [21/Sep/2024:17:16:35] ENGINE Bus STARTED 2024-09-21T17:16:36.824 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:36 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:36.824 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:36 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:36.824 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:36 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/903361819' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:37.559 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:37.560 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:37.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:37 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:37.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:37 smithi018 ceph-mon[23738]: mgrmap e17: smithi018.jndyyn(active, since 2s) 2024-09-21T17:16:39.868 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:39.868 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:39.870 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:40.302 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:39 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/3082407733' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:41.711 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:41.711 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:42.381 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:42.381 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:42.381 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd/host:smithi157", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:16:43.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd/host:smithi018", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi157", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi157", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T17:16:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:16:43.718 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:43.718 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:43.721 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:44.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:44 smithi018 ceph-mon[23738]: Deploying daemon crash.smithi157 on smithi157 2024-09-21T17:16:44.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:44 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/1288730268' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:45.421 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:45.421 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:47.520 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:47.520 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:47.522 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:48.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:47 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/3145429021' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:49.365 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:49.366 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:49.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:48 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:49.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:48 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:16:49.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:48 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T17:16:49.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:48 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:16:49.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:48 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:16:49.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:48 smithi018 ceph-mon[23738]: Deploying daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:16:50.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:50 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:51.397 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:16:51.397 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":1,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:14:43.780044Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T17:16:51.398 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 1 2024-09-21T17:16:51.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:51 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/2082923882' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:16:53.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:52 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:16:53.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:52 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T17:16:53.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:52 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:16:53.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:52 smithi018 ceph-mon[23738]: Deploying daemon mon.smithi157 on smithi157 2024-09-21T17:16:53.265 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T17:16:53.265 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T17:16:55.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:55 smithi018 ceph-mon[23738]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:16:55.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:16:55 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:01.591 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:17:01.591 INFO:teuthology.orchestra.run.smithi157.stdout:{"epoch":2,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","modified":"2024-09-21T17:16:56.579645Z","created":"2024-09-21T17:14:43.780044Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi018","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi157","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:3300","nonce":0},{"type":"v1","addr":"172.21.15.157:6789","nonce":0}]},"addr":"172.21.15.157:6789/0","public_addr":"172.21.15.157:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T17:17:01.594 INFO:teuthology.orchestra.run.smithi157.stderr:dumped monmap epoch 2 2024-09-21T17:17:01.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:01.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:17:01.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:17:01.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: mon.smithi018 calling monitor election 2024-09-21T17:17:01.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:17:01.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/4291086370' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T17:17:01.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:17:01.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/4196737937' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: mon.smithi157 calling monitor election 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: mon.smithi018 is new leader, mons smithi018,smithi157 in quorum (ranks 0,1) 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: monmap e2: 2 mons at {smithi018=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],smithi157=[v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0]} 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: fsmap 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: mgrmap e17: smithi018.jndyyn(active, since 27s) 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:17:01.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/4196737937' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:17:01.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: overall HEALTH_OK 2024-09-21T17:17:01.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/4196737937' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:17:01.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/4196737937' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:17:01.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:02.356 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T17:17:02.356 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T17:17:02.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:02 smithi018 ceph-mon[23738]: Deploying daemon node-exporter.smithi157 on smithi157 2024-09-21T17:17:02.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:02 smithi018 ceph-mon[23738]: mgrmap e18: smithi018.jndyyn(active, since 27s), standbys: smithi157.oowooq 2024-09-21T17:17:02.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:02 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi157.oowooq", "id": "smithi157.oowooq"}]: dispatch 2024-09-21T17:17:02.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:02 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:17:04.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:03 smithi018 ceph-mon[23738]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:04.272 INFO:teuthology.orchestra.run.smithi018.stdout:# minimal ceph.conf for ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:17:04.272 INFO:teuthology.orchestra.run.smithi018.stdout:[global] 2024-09-21T17:17:04.272 INFO:teuthology.orchestra.run.smithi018.stdout: fsid = ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:17:04.272 INFO:teuthology.orchestra.run.smithi018.stdout: mon_host = [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] 2024-09-21T17:17:04.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:03 smithi157 ceph-mon[25926]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:04.898 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T17:17:04.899 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:17:04.899 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T17:17:04.927 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:17:04.928 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:17:04.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:04 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2970218333' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:04.991 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:04 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:04.995 DEBUG:teuthology.orchestra.run.smithi157:> set -ex 2024-09-21T17:17:04.995 DEBUG:teuthology.orchestra.run.smithi157:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T17:17:05.019 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:04 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2970218333' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:05.020 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:04 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:05.024 DEBUG:teuthology.orchestra.run.smithi157:> set -ex 2024-09-21T17:17:05.024 DEBUG:teuthology.orchestra.run.smithi157:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:17:05.090 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T17:17:05.091 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T17:17:05.091 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:17:05.091 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T17:17:05.109 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T17:17:05.109 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_1 2024-09-21T17:17:05.164 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T17:17:05.164 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T17:17:05.164 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-21T17:17:05.164 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T17:17:05.164 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T17:17:05.164 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2024-09-21 17:16:24.413771507 +0000 2024-09-21T17:17:05.164 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2024-09-21 17:13:54.609542602 +0000 2024-09-21T17:17:05.165 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2024-09-21 17:13:54.609542602 +0000 2024-09-21T17:17:05.165 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: 2024-09-21 17:13:54.609542602 +0000 2024-09-21T17:17:05.165 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T17:17:05.229 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2024-09-21T17:17:05.229 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2024-09-21T17:17:05.229 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000113504 s, 4.5 MB/s 2024-09-21T17:17:05.230 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T17:17:05.287 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_2 2024-09-21T17:17:05.343 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T17:17:05.343 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T17:17:05.343 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-21T17:17:05.343 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T17:17:05.344 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T17:17:05.344 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2024-09-21 17:16:24.413771507 +0000 2024-09-21T17:17:05.344 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2024-09-21 17:13:54.937538969 +0000 2024-09-21T17:17:05.344 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2024-09-21 17:13:54.937538969 +0000 2024-09-21T17:17:05.344 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: 2024-09-21 17:13:54.937538969 +0000 2024-09-21T17:17:05.344 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T17:17:05.409 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2024-09-21T17:17:05.409 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2024-09-21T17:17:05.409 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 9.3121e-05 s, 5.5 MB/s 2024-09-21T17:17:05.410 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T17:17:05.466 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_3 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2024-09-21 17:16:24.413771507 +0000 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2024-09-21 17:13:55.254535457 +0000 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2024-09-21 17:13:55.254535457 +0000 2024-09-21T17:17:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: 2024-09-21 17:13:55.254535457 +0000 2024-09-21T17:17:05.523 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T17:17:05.587 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2024-09-21T17:17:05.588 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2024-09-21T17:17:05.588 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000113316 s, 4.5 MB/s 2024-09-21T17:17:05.589 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T17:17:05.645 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_4 2024-09-21T17:17:05.703 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T17:17:05.703 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T17:17:05.704 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-21T17:17:05.704 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T17:17:05.704 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T17:17:05.704 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2024-09-21 17:16:24.414771496 +0000 2024-09-21T17:17:05.704 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2024-09-21 17:13:55.576531890 +0000 2024-09-21T17:17:05.704 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2024-09-21 17:13:55.576531890 +0000 2024-09-21T17:17:05.704 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: 2024-09-21 17:13:55.576531890 +0000 2024-09-21T17:17:05.705 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T17:17:05.768 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2024-09-21T17:17:05.768 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2024-09-21T17:17:05.768 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000104935 s, 4.9 MB/s 2024-09-21T17:17:05.769 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T17:17:05.827 DEBUG:teuthology.orchestra.run.smithi157:> set -ex 2024-09-21T17:17:05.828 DEBUG:teuthology.orchestra.run.smithi157:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T17:17:05.844 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T17:17:05.844 DEBUG:teuthology.orchestra.run.smithi157:> stat /dev/vg_nvme/lv_1 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout:Access: 2024-09-21 17:17:02.747753211 +0000 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout:Modify: 2024-09-21 17:13:48.025689741 +0000 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout:Change: 2024-09-21 17:13:48.025689741 +0000 2024-09-21T17:17:05.901 INFO:teuthology.orchestra.run.smithi157.stdout: Birth: 2024-09-21 17:13:48.025689741 +0000 2024-09-21T17:17:05.902 DEBUG:teuthology.orchestra.run.smithi157:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T17:17:05.965 INFO:teuthology.orchestra.run.smithi157.stderr:1+0 records in 2024-09-21T17:17:05.966 INFO:teuthology.orchestra.run.smithi157.stderr:1+0 records out 2024-09-21T17:17:05.966 INFO:teuthology.orchestra.run.smithi157.stderr:512 bytes copied, 0.000122095 s, 4.2 MB/s 2024-09-21T17:17:05.967 DEBUG:teuthology.orchestra.run.smithi157:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T17:17:06.023 DEBUG:teuthology.orchestra.run.smithi157:> stat /dev/vg_nvme/lv_2 2024-09-21T17:17:06.079 INFO:teuthology.orchestra.run.smithi157.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T17:17:06.079 INFO:teuthology.orchestra.run.smithi157.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T17:17:06.079 INFO:teuthology.orchestra.run.smithi157.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-21T17:17:06.079 INFO:teuthology.orchestra.run.smithi157.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T17:17:06.079 INFO:teuthology.orchestra.run.smithi157.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T17:17:06.079 INFO:teuthology.orchestra.run.smithi157.stdout:Access: 2024-09-21 17:17:02.747753211 +0000 2024-09-21T17:17:06.080 INFO:teuthology.orchestra.run.smithi157.stdout:Modify: 2024-09-21 17:13:48.336683969 +0000 2024-09-21T17:17:06.080 INFO:teuthology.orchestra.run.smithi157.stdout:Change: 2024-09-21 17:13:48.337683950 +0000 2024-09-21T17:17:06.080 INFO:teuthology.orchestra.run.smithi157.stdout: Birth: 2024-09-21 17:13:48.336683969 +0000 2024-09-21T17:17:06.080 DEBUG:teuthology.orchestra.run.smithi157:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T17:17:06.141 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:05 smithi157 ceph-mon[25926]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:06.141 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:05 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:06.143 INFO:teuthology.orchestra.run.smithi157.stderr:1+0 records in 2024-09-21T17:17:06.143 INFO:teuthology.orchestra.run.smithi157.stderr:1+0 records out 2024-09-21T17:17:06.143 INFO:teuthology.orchestra.run.smithi157.stderr:512 bytes copied, 0.000110627 s, 4.6 MB/s 2024-09-21T17:17:06.144 DEBUG:teuthology.orchestra.run.smithi157:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T17:17:06.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:05 smithi018 ceph-mon[23738]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:06.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:05 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:06.201 DEBUG:teuthology.orchestra.run.smithi157:> stat /dev/vg_nvme/lv_3 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout:Access: 2024-09-21 17:17:02.747753211 +0000 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout:Modify: 2024-09-21 17:13:48.658677992 +0000 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout:Change: 2024-09-21 17:13:48.658677992 +0000 2024-09-21T17:17:06.256 INFO:teuthology.orchestra.run.smithi157.stdout: Birth: 2024-09-21 17:13:48.658677992 +0000 2024-09-21T17:17:06.257 DEBUG:teuthology.orchestra.run.smithi157:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T17:17:06.319 INFO:teuthology.orchestra.run.smithi157.stderr:1+0 records in 2024-09-21T17:17:06.319 INFO:teuthology.orchestra.run.smithi157.stderr:1+0 records out 2024-09-21T17:17:06.319 INFO:teuthology.orchestra.run.smithi157.stderr:512 bytes copied, 0.000111254 s, 4.6 MB/s 2024-09-21T17:17:06.321 DEBUG:teuthology.orchestra.run.smithi157:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T17:17:06.377 DEBUG:teuthology.orchestra.run.smithi157:> stat /dev/vg_nvme/lv_4 2024-09-21T17:17:06.432 INFO:teuthology.orchestra.run.smithi157.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T17:17:06.432 INFO:teuthology.orchestra.run.smithi157.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T17:17:06.432 INFO:teuthology.orchestra.run.smithi157.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-21T17:17:06.432 INFO:teuthology.orchestra.run.smithi157.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T17:17:06.432 INFO:teuthology.orchestra.run.smithi157.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T17:17:06.432 INFO:teuthology.orchestra.run.smithi157.stdout:Access: 2024-09-21 17:17:02.748753193 +0000 2024-09-21T17:17:06.432 INFO:teuthology.orchestra.run.smithi157.stdout:Modify: 2024-09-21 17:13:48.965672293 +0000 2024-09-21T17:17:06.432 INFO:teuthology.orchestra.run.smithi157.stdout:Change: 2024-09-21 17:13:48.965672293 +0000 2024-09-21T17:17:06.433 INFO:teuthology.orchestra.run.smithi157.stdout: Birth: 2024-09-21 17:13:48.965672293 +0000 2024-09-21T17:17:06.433 DEBUG:teuthology.orchestra.run.smithi157:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T17:17:06.495 INFO:teuthology.orchestra.run.smithi157.stderr:1+0 records in 2024-09-21T17:17:06.495 INFO:teuthology.orchestra.run.smithi157.stderr:1+0 records out 2024-09-21T17:17:06.495 INFO:teuthology.orchestra.run.smithi157.stderr:512 bytes copied, 8.3663e-05 s, 6.1 MB/s 2024-09-21T17:17:06.496 DEBUG:teuthology.orchestra.run.smithi157:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T17:17:06.553 INFO:tasks.cephadm:Deploying osd.0 on smithi018 with /dev/vg_nvme/lv_4... 2024-09-21T17:17:06.554 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T17:17:07.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:06 smithi018 ceph-mon[23738]: Deploying daemon prometheus.smithi018 on smithi018 2024-09-21T17:17:07.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:06 smithi157 ceph-mon[25926]: Deploying daemon prometheus.smithi018 on smithi018 2024-09-21T17:17:08.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:07 smithi018 ceph-mon[23738]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:08.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:07 smithi157 ceph-mon[25926]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:10.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:09 smithi018 ceph-mon[23738]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:10.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:09 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:10.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:09 smithi157 ceph-mon[25926]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:10.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:09 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:12.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:11 smithi018 ceph-mon[23738]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:12.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:11 smithi157 ceph-mon[25926]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:14.091 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:13 smithi018 ceph-mon[23738]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:14.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:13 smithi157 ceph-mon[25926]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:15.981 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:15 smithi018 ceph-mon[23738]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:16.175 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:15 smithi157 ceph-mon[25926]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:17:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:17:17.517 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:17.518 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:17:17.518 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:17.518 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:17:17.857 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:17:17.881 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi018:vg_nvme/lv_4 2024-09-21T17:17:18.294 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:18 smithi157 ceph-mon[25926]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:18.294 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:18 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:18.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:18 smithi018 ceph-mon[23738]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:18.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:18 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:19 smithi018 ceph-mon[23738]: Updating smithi157:/etc/ceph/ceph.conf 2024-09-21T17:17:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:19 smithi018 ceph-mon[23738]: Updating smithi157:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:17:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:19 smithi157 ceph-mon[25926]: Updating smithi157:/etc/ceph/ceph.conf 2024-09-21T17:17:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:19 smithi157 ceph-mon[25926]: Updating smithi157:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:17:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:20.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:20 smithi018 ceph-mon[23738]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:20.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:20 smithi157 ceph-mon[25926]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:21 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:17:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:21 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:17:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:21 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:21 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:17:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:21 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:17:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:21 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:22.429 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:22 smithi157 ceph-mon[25926]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:22.429 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:22 smithi157 ceph-mon[25926]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:17:22.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:22 smithi018 ceph-mon[23738]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:22.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:22 smithi018 ceph-mon[23738]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:17:24.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:24 smithi018 ceph-mon[23738]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:24 smithi157 ceph-mon[25926]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:17:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: Reconfiguring mon.smithi018 (unknown last config time)... 2024-09-21T17:17:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T17:17:25.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T17:17:25.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:25.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:25 smithi018 ceph-mon[23738]: Reconfiguring daemon mon.smithi018 on smithi018 2024-09-21T17:17:25.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:25.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:17:25.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:25.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:25.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:25.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: Reconfiguring mon.smithi018 (unknown last config time)... 2024-09-21T17:17:25.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T17:17:25.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T17:17:25.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:25.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:25 smithi157 ceph-mon[25926]: Reconfiguring daemon mon.smithi018 on smithi018 2024-09-21T17:17:26.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:26 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/1001988968' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac9a72bf-1285-4f67-b0a4-0e39841eae95"}]: dispatch 2024-09-21T17:17:26.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:26 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/1001988968' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac9a72bf-1285-4f67-b0a4-0e39841eae95"}]': finished 2024-09-21T17:17:26.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:26 smithi157 ceph-mon[25926]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T17:17:26.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:26 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:26.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:26 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1001988968' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac9a72bf-1285-4f67-b0a4-0e39841eae95"}]: dispatch 2024-09-21T17:17:26.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:26 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1001988968' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac9a72bf-1285-4f67-b0a4-0e39841eae95"}]': finished 2024-09-21T17:17:26.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:26 smithi018 ceph-mon[23738]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T17:17:26.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:26 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:27 smithi157 ceph-mon[25926]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:27 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/1037109557' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:17:27.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:27 smithi018 ceph-mon[23738]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:27.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:27 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1037109557' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:17:29.709 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:29 smithi018 ceph-mon[23738]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:30.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:29 smithi157 ceph-mon[25926]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:31.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:31 smithi018 ceph-mon[23738]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:31 smithi157 ceph-mon[25926]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:32.944 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:32.944 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:32 smithi157 ceph-mon[25926]: Reconfiguring mgr.smithi018.jndyyn (unknown last config time)... 2024-09-21T17:17:32.944 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi018.jndyyn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:17:32.944 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:17:32.944 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:32.944 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:32 smithi157 ceph-mon[25926]: Reconfiguring daemon mgr.smithi018.jndyyn on smithi018 2024-09-21T17:17:33.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:33.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:32 smithi018 ceph-mon[23738]: Reconfiguring mgr.smithi018.jndyyn (unknown last config time)... 2024-09-21T17:17:33.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi018.jndyyn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:17:33.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:17:33.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:33.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:32 smithi018 ceph-mon[23738]: Reconfiguring daemon mgr.smithi018.jndyyn on smithi018 2024-09-21T17:17:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:33 smithi157 ceph-mon[25926]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:33 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T17:17:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:33 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:34.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:33 smithi157 ceph-mon[25926]: Deploying daemon osd.0 on smithi018 2024-09-21T17:17:34.060 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:33 smithi018 ceph-mon[23738]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:34.060 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:33 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T17:17:34.060 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:33 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:34.060 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:33 smithi018 ceph-mon[23738]: Deploying daemon osd.0 on smithi018 2024-09-21T17:17:35.487 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:35.487 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:35 smithi018 ceph-mon[23738]: Reconfiguring alertmanager.smithi018 (dependencies changed)... 2024-09-21T17:17:35.487 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:35 smithi018 ceph-mon[23738]: Reconfiguring daemon alertmanager.smithi018 on smithi018 2024-09-21T17:17:35.487 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:35 smithi018 ceph-mon[23738]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:35.487 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:17:35.488 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:17:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:35 smithi157 ceph-mon[25926]: Reconfiguring alertmanager.smithi018 (dependencies changed)... 2024-09-21T17:17:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:35 smithi157 ceph-mon[25926]: Reconfiguring daemon alertmanager.smithi018 on smithi018 2024-09-21T17:17:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:35 smithi157 ceph-mon[25926]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:17:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:17:38.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:37 smithi018 ceph-mon[23738]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:38.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:37 smithi157 ceph-mon[25926]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:40.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:39 smithi018 ceph-mon[23738]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:40.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:39 smithi157 ceph-mon[25926]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:42.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:41 smithi018 ceph-mon[23738]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:41 smithi157 ceph-mon[25926]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:43.053 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:42 smithi018 ceph-mon[23738]: from='osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T17:17:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:42 smithi157 ceph-mon[25926]: from='osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T17:17:44.058 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:43 smithi018 ceph-mon[23738]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:44.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:43 smithi018 ceph-mon[23738]: from='osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T17:17:44.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:43 smithi018 ceph-mon[23738]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T17:17:44.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:44.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:43 smithi018 ceph-mon[23738]: from='osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T17:17:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:43 smithi157 ceph-mon[25926]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:43 smithi157 ceph-mon[25926]: from='osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T17:17:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:43 smithi157 ceph-mon[25926]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T17:17:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:43 smithi157 ceph-mon[25926]: from='osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T17:17:45.074 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:44 smithi018 ceph-mon[23738]: from='osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-21T17:17:45.074 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:44 smithi018 ceph-mon[23738]: osdmap e8: 1 total, 0 up, 1 in 2024-09-21T17:17:45.074 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:44 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:45.074 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:44 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:45.074 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:44 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:45.074 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:44 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi018", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T17:17:45.074 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:44 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:44 smithi157 ceph-mon[25926]: from='osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-21T17:17:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:44 smithi157 ceph-mon[25926]: osdmap e8: 1 total, 0 up, 1 in 2024-09-21T17:17:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:44 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:44 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:45.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:44 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:45.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:44 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi018", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T17:17:45.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:44 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:46.127 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: purged_snaps scrub starts 2024-09-21T17:17:46.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: purged_snaps scrub ok 2024-09-21T17:17:46.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:46.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: Reconfiguring crash.smithi018 (monmap changed)... 2024-09-21T17:17:46.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: Reconfiguring daemon crash.smithi018 on smithi018 2024-09-21T17:17:46.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:46.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795] boot 2024-09-21T17:17:46.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T17:17:46.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:45 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:46.170 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 0 on host 'smithi018' 2024-09-21T17:17:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: purged_snaps scrub starts 2024-09-21T17:17:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: purged_snaps scrub ok 2024-09-21T17:17:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T17:17:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: Reconfiguring crash.smithi018 (monmap changed)... 2024-09-21T17:17:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: Reconfiguring daemon crash.smithi018 on smithi018 2024-09-21T17:17:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: osd.0 [v2:172.21.15.18:6802/2412849795,v1:172.21.15.18:6803/2412849795] boot 2024-09-21T17:17:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T17:17:46.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:45 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:17:46.877 DEBUG:teuthology.orchestra.run.smithi018:osd.0> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T17:17:46.879 INFO:tasks.cephadm:Deploying osd.1 on smithi018 with /dev/vg_nvme/lv_3... 2024-09-21T17:17:46.879 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T17:17:47.122 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:46 smithi018 ceph-mon[23738]: osdmap e10: 1 total, 1 up, 1 in 2024-09-21T17:17:47.122 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:47.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:46 smithi157 ceph-mon[25926]: osdmap e10: 1 total, 1 up, 1 in 2024-09-21T17:17:47.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:48 smithi018 ceph-mon[23738]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:48 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:48 smithi157 ceph-mon[25926]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:48 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:49.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:49 smithi018 ceph-mon[23738]: Reconfiguring grafana.smithi018 (dependencies changed)... 2024-09-21T17:17:49.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:49 smithi018 ceph-mon[23738]: Reconfiguring daemon grafana.smithi018 on smithi018 2024-09-21T17:17:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:49 smithi157 ceph-mon[25926]: Reconfiguring grafana.smithi018 (dependencies changed)... 2024-09-21T17:17:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:49 smithi157 ceph-mon[25926]: Reconfiguring daemon grafana.smithi018 on smithi018 2024-09-21T17:17:50.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:50 smithi157 ceph-mon[25926]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:50.529 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:50 smithi018 ceph-mon[23738]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:52.497 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:52 smithi018 ceph-mon[23738]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:52 smithi157 ceph-mon[25926]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:54.031 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:53 smithi018 ceph-mon[23738]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:54.031 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:53 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:54.031 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:53 smithi018 ceph-mon[23738]: Reconfiguring crash.smithi157 (monmap changed)... 2024-09-21T17:17:54.031 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:53 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi157", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T17:17:54.031 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:53 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:54.031 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:53 smithi018 ceph-mon[23738]: Reconfiguring daemon crash.smithi157 on smithi157 2024-09-21T17:17:54.250 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:53 smithi157 ceph-mon[25926]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:54.250 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:53 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:54.250 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:53 smithi157 ceph-mon[25926]: Reconfiguring crash.smithi157 (monmap changed)... 2024-09-21T17:17:54.250 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:53 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi157", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T17:17:54.251 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:53 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:54.251 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:53 smithi157 ceph-mon[25926]: Reconfiguring daemon crash.smithi157 on smithi157 2024-09-21T17:17:54.362 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:17:54.384 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi018:vg_nvme/lv_3 2024-09-21T17:17:55.606 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:55 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:55.606 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:55 smithi157 ceph-mon[25926]: Reconfiguring mgr.smithi157.oowooq (monmap changed)... 2024-09-21T17:17:55.607 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:55 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:17:55.607 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:55 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:17:55.607 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:55 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:55.607 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:55 smithi157 ceph-mon[25926]: Reconfiguring daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:17:55.607 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:55 smithi157 ceph-mon[25926]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:55.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:55 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:55.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:55 smithi018 ceph-mon[23738]: Reconfiguring mgr.smithi157.oowooq (monmap changed)... 2024-09-21T17:17:55.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:55 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:17:55.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:55 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:17:55.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:55 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:55.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:55 smithi018 ceph-mon[23738]: Reconfiguring daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:17:55.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:55 smithi018 ceph-mon[23738]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:56.969 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: Reconfiguring mon.smithi157 (monmap changed)... 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: Reconfiguring daemon mon.smithi157 on smithi157 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:56.970 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:56.971 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: Reconfiguring mon.smithi157 (monmap changed)... 2024-09-21T17:17:56.971 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T17:17:56.971 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T17:17:56.971 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:56.971 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: Reconfiguring daemon mon.smithi157 on smithi157 2024-09-21T17:17:56.971 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:17:56.971 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:17:56.971 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:17:58.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:58 smithi157 ceph-mon[25926]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:17:58.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:58 smithi157 ceph-mon[25926]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:58.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:58 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:58.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:58 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T17:17:58.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:58 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T17:17:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:58 smithi018 ceph-mon[23738]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:17:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:58 smithi018 ceph-mon[23738]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:17:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:58 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:17:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:58 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T17:17:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:58 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T17:17:59.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:59 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3877531739' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd92a0a6-7111-4ed3-b70b-20c6dd0a45df"}]: dispatch 2024-09-21T17:17:59.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:59 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3877531739' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bd92a0a6-7111-4ed3-b70b-20c6dd0a45df"}]': finished 2024-09-21T17:17:59.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:59 smithi157 ceph-mon[25926]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T17:17:59.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:17:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:17:59.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:59 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3877531739' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bd92a0a6-7111-4ed3-b70b-20c6dd0a45df"}]: dispatch 2024-09-21T17:17:59.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:59 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3877531739' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bd92a0a6-7111-4ed3-b70b-20c6dd0a45df"}]': finished 2024-09-21T17:17:59.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:59 smithi018 ceph-mon[23738]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T17:17:59.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:17:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:00.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:00 smithi157 ceph-mon[25926]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:00.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:00 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/149608500' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:18:00.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:00 smithi018 ceph-mon[23738]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:00.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:00 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/149608500' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:18:02.231 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:02 smithi018 ceph-mon[23738]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:02 smithi157 ceph-mon[25926]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:04.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:04 smithi018 ceph-mon[23738]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:04.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:04 smithi157 ceph-mon[25926]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:05.145 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:05 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T17:18:05.145 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:05 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:05 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T17:18:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:05 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:06.376 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:06 smithi018 ceph-mon[23738]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:06.376 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:06 smithi018 ceph-mon[23738]: Deploying daemon osd.1 on smithi018 2024-09-21T17:18:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:06 smithi157 ceph-mon[25926]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:06 smithi157 ceph-mon[25926]: Deploying daemon osd.1 on smithi018 2024-09-21T17:18:08.378 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:08 smithi018 ceph-mon[23738]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:08 smithi157 ceph-mon[25926]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:10.294 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:10 smithi018 ceph-mon[23738]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:10.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:10 smithi157 ceph-mon[25926]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:11.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:11 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:11 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:12.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:12 smithi018 ceph-mon[23738]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:12 smithi157 ceph-mon[25926]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:12.980 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 1 on host 'smithi018' 2024-09-21T17:18:13.651 DEBUG:teuthology.orchestra.run.smithi018:osd.1> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T17:18:13.654 INFO:tasks.cephadm:Deploying osd.2 on smithi018 with /dev/vg_nvme/lv_2... 2024-09-21T17:18:13.655 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T17:18:14.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:14.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:14.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-21T17:18:14.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-21T17:18:14.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:14.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:18:14.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:18:14.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.18:3000"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.18:3000"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.18:9095"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.18:9095"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:18:14.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:13 smithi018 ceph-mon[23738]: from='osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T17:18:14.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:14.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:14.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-21T17:18:14.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.18:9093"}]: dispatch 2024-09-21T17:18:14.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:14.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.18:3000"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.18:3000"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.18:9095"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.18:9095"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:18:14.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:13 smithi157 ceph-mon[25926]: from='osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T17:18:15.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:15 smithi018 ceph-mon[23738]: from='osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T17:18:15.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:15 smithi018 ceph-mon[23738]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T17:18:15.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:15 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:15.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:15 smithi018 ceph-mon[23738]: from='osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T17:18:15.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:15 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:15.404 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:18:15 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[37035]: 2024-09-21T17:18:15.151+0000 7fb239e74700 -1 osd.1 0 waiting for initial osdmap 2024-09-21T17:18:15.405 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:18:15 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[37035]: 2024-09-21T17:18:15.155+0000 7fb232807700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:18:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:15 smithi157 ceph-mon[25926]: from='osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T17:18:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:15 smithi157 ceph-mon[25926]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T17:18:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:15 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:15 smithi157 ceph-mon[25926]: from='osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T17:18:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:15 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:16.415 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:16 smithi018 ceph-mon[23738]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:16.415 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:16 smithi018 ceph-mon[23738]: from='osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-21T17:18:16.415 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:16 smithi018 ceph-mon[23738]: osdmap e13: 2 total, 1 up, 2 in 2024-09-21T17:18:16.415 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:16 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:16.415 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:16 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:16 smithi157 ceph-mon[25926]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:16 smithi157 ceph-mon[25926]: from='osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-21T17:18:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:16 smithi157 ceph-mon[25926]: osdmap e13: 2 total, 1 up, 2 in 2024-09-21T17:18:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:16 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:16 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:17.089 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:18:17.113 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi018:vg_nvme/lv_2 2024-09-21T17:18:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:17 smithi018 ceph-mon[23738]: purged_snaps scrub starts 2024-09-21T17:18:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:17 smithi018 ceph-mon[23738]: purged_snaps scrub ok 2024-09-21T17:18:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:17 smithi018 ceph-mon[23738]: osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049] boot 2024-09-21T17:18:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:17 smithi018 ceph-mon[23738]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T17:18:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:17 smithi157 ceph-mon[25926]: purged_snaps scrub starts 2024-09-21T17:18:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:17 smithi157 ceph-mon[25926]: purged_snaps scrub ok 2024-09-21T17:18:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:17 smithi157 ceph-mon[25926]: osd.1 [v2:172.21.15.18:6810/3829486049,v1:172.21.15.18:6811/3829486049] boot 2024-09-21T17:18:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:17 smithi157 ceph-mon[25926]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T17:18:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:18:18.161 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:18 smithi018 ceph-mon[23738]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:18.161 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:18 smithi018 ceph-mon[23738]: osdmap e15: 2 total, 2 up, 2 in 2024-09-21T17:18:18.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:18 smithi157 ceph-mon[25926]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T17:18:18.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:18 smithi157 ceph-mon[25926]: osdmap e15: 2 total, 2 up, 2 in 2024-09-21T17:18:20.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:20 smithi018 ceph-mon[23738]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:20.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:18:20.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:18:20.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:20 smithi157 ceph-mon[25926]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:18:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:18:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:21.459 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:21 smithi018 ceph-mon[23738]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:18:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:21 smithi157 ceph-mon[25926]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:18:22.429 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:22 smithi157 ceph-mon[25926]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:22.429 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:22 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:22.469 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:22 smithi018 ceph-mon[23738]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:22.469 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:22 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:24.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:24 smithi018 ceph-mon[23738]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:24.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:24 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3845424137' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40e5c057-8087-4fbf-9d48-37a163accaad"}]: dispatch 2024-09-21T17:18:24.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:24 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3845424137' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40e5c057-8087-4fbf-9d48-37a163accaad"}]': finished 2024-09-21T17:18:24.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:24 smithi018 ceph-mon[23738]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T17:18:24.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:24 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:24 smithi157 ceph-mon[25926]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:24 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3845424137' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40e5c057-8087-4fbf-9d48-37a163accaad"}]: dispatch 2024-09-21T17:18:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:24 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3845424137' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40e5c057-8087-4fbf-9d48-37a163accaad"}]': finished 2024-09-21T17:18:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:24 smithi157 ceph-mon[25926]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T17:18:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:24 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:25 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3107217885' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:18:25.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:25 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3107217885' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:18:26.429 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:26 smithi018 ceph-mon[23738]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:26 smithi157 ceph-mon[25926]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:28.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:28 smithi018 ceph-mon[23738]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:28 smithi157 ceph-mon[25926]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:30.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:30.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: Detected new or changed devices on smithi018 2024-09-21T17:18:30.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:30.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:30.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:30.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi018 to 6434M 2024-09-21T17:18:30.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:30.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:30.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:18:30.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:30.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:30 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: Detected new or changed devices on smithi018 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi018 to 6434M 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:30.296 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:30 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:18:32.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:31 smithi018 ceph-mon[23738]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:32.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:31 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:32.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:31 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T17:18:32.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:31 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:32.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:31 smithi018 ceph-mon[23738]: Deploying daemon osd.2 on smithi018 2024-09-21T17:18:32.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:31 smithi157 ceph-mon[25926]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:32.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:31 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:32.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:31 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T17:18:32.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:31 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:32.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:31 smithi157 ceph-mon[25926]: Deploying daemon osd.2 on smithi018 2024-09-21T17:18:34.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:33 smithi018 ceph-mon[23738]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:34.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:33 smithi157 ceph-mon[25926]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:18:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:18:35.125 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:18:35.125 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:18:36.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:35 smithi018 ceph-mon[23738]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:36.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:35 smithi157 ceph-mon[25926]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:38.033 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:37 smithi018 ceph-mon[23738]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:38.033 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:37 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:38.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:37 smithi157 ceph-mon[25926]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:38.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:37 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:39.415 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:39.415 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:39 smithi018 ceph-mon[23738]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:39.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:39.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:39 smithi157 ceph-mon[25926]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:41.889 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:41 smithi018 ceph-mon[23738]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:41.889 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:41 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:41.889 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:41 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:41.889 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:41 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:18:41.889 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:41 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:41.890 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:41 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:18:41.890 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:41 smithi018 ceph-mon[23738]: from='osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T17:18:41.914 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:41 smithi157 ceph-mon[25926]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:41.914 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:41 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:41.914 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:41 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:41.914 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:41 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:18:41.914 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:41 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:41.914 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:41 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:18:41.914 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:41 smithi157 ceph-mon[25926]: from='osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T17:18:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:43 smithi018 ceph-mon[23738]: from='osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T17:18:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:43 smithi018 ceph-mon[23738]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T17:18:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:43 smithi018 ceph-mon[23738]: from='osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T17:18:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:43 smithi018 ceph-mon[23738]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:43.479 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 2 on host 'smithi018' 2024-09-21T17:18:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:43 smithi157 ceph-mon[25926]: from='osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T17:18:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:43 smithi157 ceph-mon[25926]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T17:18:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:43 smithi157 ceph-mon[25926]: from='osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T17:18:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:43 smithi157 ceph-mon[25926]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:44.452 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:44 smithi018 ceph-mon[23738]: from='osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-21T17:18:44.452 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:44 smithi018 ceph-mon[23738]: osdmap e18: 3 total, 2 up, 3 in 2024-09-21T17:18:44.452 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:44 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:44.452 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:44 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:44.452 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:44 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:44.476 DEBUG:teuthology.orchestra.run.smithi018:osd.2> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.2.service 2024-09-21T17:18:44.478 INFO:tasks.cephadm:Deploying osd.3 on smithi018 with /dev/vg_nvme/lv_1... 2024-09-21T17:18:44.478 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T17:18:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:44 smithi157 ceph-mon[25926]: from='osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-21T17:18:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:44 smithi157 ceph-mon[25926]: osdmap e18: 3 total, 2 up, 3 in 2024-09-21T17:18:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:44 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:44 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:44.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:44 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:45 smithi157 ceph-mon[25926]: purged_snaps scrub starts 2024-09-21T17:18:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:45 smithi157 ceph-mon[25926]: purged_snaps scrub ok 2024-09-21T17:18:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:45 smithi157 ceph-mon[25926]: osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202] boot 2024-09-21T17:18:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:45 smithi157 ceph-mon[25926]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T17:18:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:45 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:45 smithi157 ceph-mon[25926]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:45.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:45 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T17:18:45.626 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:45 smithi018 ceph-mon[23738]: purged_snaps scrub starts 2024-09-21T17:18:45.626 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:45 smithi018 ceph-mon[23738]: purged_snaps scrub ok 2024-09-21T17:18:45.626 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:45 smithi018 ceph-mon[23738]: osd.2 [v2:172.21.15.18:6818/2209132202,v1:172.21.15.18:6819/2209132202] boot 2024-09-21T17:18:45.626 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:45 smithi018 ceph-mon[23738]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T17:18:45.626 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:45 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:18:45.626 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:45 smithi018 ceph-mon[23738]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T17:18:45.627 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:45 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T17:18:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T17:18:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:46 smithi157 ceph-mon[25926]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T17:18:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T17:18:46.540 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T17:18:46.540 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:46 smithi018 ceph-mon[23738]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T17:18:46.540 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T17:18:47.152 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:18:46 smithi018 sudo[42059]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T17:18:47.153 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:18:46 smithi018 sudo[42059]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:47.153 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:18:46 smithi018 sudo[42059]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:47 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T17:18:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:47 smithi157 ceph-mon[25926]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T17:18:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:47 smithi157 ceph-mon[25926]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:18:47.570 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:18:47 smithi018 sudo[42110]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T17:18:47.570 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:18:47 smithi018 sudo[42110]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:47.570 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:18:47 smithi018 sudo[42110]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:47.571 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:47 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T17:18:47.571 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:47 smithi018 ceph-mon[23738]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T17:18:47.571 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:47 smithi018 ceph-mon[23738]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:18:47.576 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:18:47.594 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi018:vg_nvme/lv_1 2024-09-21T17:18:48.123 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:18:47 smithi018 sudo[42155]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T17:18:48.123 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:18:47 smithi018 sudo[42155]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:48.123 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:18:47 smithi018 sudo[42155]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:48.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:48 smithi018 ceph-mon[23738]: mgrmap e19: smithi018.jndyyn(active, since 2m), standbys: smithi157.oowooq 2024-09-21T17:18:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:48 smithi018 ceph-mon[23738]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T17:18:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:48 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:48.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:18:48 smithi018 sudo[42195]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T17:18:48.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:18:48 smithi018 sudo[42195]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:48.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:18:48 smithi018 sudo[42195]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:48 smithi157 ceph-mon[25926]: mgrmap e19: smithi018.jndyyn(active, since 2m), standbys: smithi157.oowooq 2024-09-21T17:18:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:48 smithi157 ceph-mon[25926]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T17:18:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:48 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:49.153 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:18:48 smithi018 sudo[42251]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T17:18:49.153 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:18:48 smithi018 sudo[42251]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:49.153 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:18:48 smithi018 sudo[42251]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:49.472 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:18:49 smithi018 sudo[42262]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T17:18:49.472 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:18:49 smithi018 sudo[42262]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:49.472 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:18:49 smithi018 sudo[42262]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:49.723 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:49 smithi018 sudo[42284]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T17:18:49.723 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:49 smithi018 sudo[42284]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:49.723 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:49 smithi018 sudo[42284]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:50.361 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 sudo[42315]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T17:18:50.361 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 sudo[42315]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:50.361 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 sudo[42315]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:50.596 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 sudo[28140]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T17:18:50.596 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 sudo[28140]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:50.596 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 sudo[28140]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:50.596 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 sudo[28143]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T17:18:50.596 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 sudo[28143]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T17:18:50.596 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 sudo[28143]: pam_unix(sudo:session): session closed for user root 2024-09-21T17:18:50.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 ceph-mon[23738]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:18:50.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 ceph-mon[23738]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T17:18:50.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 ceph-mon[23738]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T17:18:50.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:18:50.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:18:50.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:18:50.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:50 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:18:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 ceph-mon[25926]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:18:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 ceph-mon[25926]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T17:18:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 ceph-mon[25926]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T17:18:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:18:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:18:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:18:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:50 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:18:51.879 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:51 smithi018 ceph-mon[23738]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T17:18:51.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:51 smithi018 ceph-mon[23738]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:18:51.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:18:51.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:18:51.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:51.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:51 smithi018 ceph-mon[23738]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:18:51.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:51 smithi018 ceph-mon[23738]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T17:18:51.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:51 smithi157 ceph-mon[25926]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T17:18:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:51 smithi157 ceph-mon[25926]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:18:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:18:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:18:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:51 smithi157 ceph-mon[25926]: pgmap v78: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:18:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:51 smithi157 ceph-mon[25926]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T17:18:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:53 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:53 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/565481110' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8be04aa4-bfcc-4407-bb16-8500a8f5ae66"}]: dispatch 2024-09-21T17:18:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:53 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/565481110' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8be04aa4-bfcc-4407-bb16-8500a8f5ae66"}]': finished 2024-09-21T17:18:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:53 smithi157 ceph-mon[25926]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T17:18:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:53 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:18:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:53 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:53 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/565481110' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8be04aa4-bfcc-4407-bb16-8500a8f5ae66"}]: dispatch 2024-09-21T17:18:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:53 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/565481110' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8be04aa4-bfcc-4407-bb16-8500a8f5ae66"}]': finished 2024-09-21T17:18:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:53 smithi018 ceph-mon[23738]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T17:18:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:53 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:18:54.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:54 smithi018 ceph-mon[23738]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 5.1 KiB/s rd, 247 KiB/s wr, 17 op/s 2024-09-21T17:18:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:54 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2578212097' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:18:54.456 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:54 smithi157 ceph-mon[25926]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 5.1 KiB/s rd, 247 KiB/s wr, 17 op/s 2024-09-21T17:18:54.456 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:54 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2578212097' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:18:56.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:56 smithi018 ceph-mon[23738]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 179 KiB/s wr, 12 op/s 2024-09-21T17:18:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:56 smithi157 ceph-mon[25926]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 179 KiB/s wr, 12 op/s 2024-09-21T17:18:58.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:58 smithi018 ceph-mon[23738]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 141 KiB/s wr, 9 op/s 2024-09-21T17:18:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:58 smithi157 ceph-mon[25926]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 141 KiB/s wr, 9 op/s 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: Detected new or changed devices on smithi018 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi018 to 4289M 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:18:59.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:18:59 smithi018 ceph-mon[23738]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-21T17:18:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: Detected new or changed devices on smithi018 2024-09-21T17:18:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:18:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi018 to 4289M 2024-09-21T17:18:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:18:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:18:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:18:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:18:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:18:59 smithi157 ceph-mon[25926]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-21T17:19:00.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:00 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T17:19:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:00 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:00.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:00 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T17:19:00.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:00 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:01.746 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:01 smithi018 ceph-mon[23738]: Deploying daemon osd.3 on smithi018 2024-09-21T17:19:01.746 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:01 smithi018 ceph-mon[23738]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-21T17:19:01.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:01 smithi157 ceph-mon[25926]: Deploying daemon osd.3 on smithi018 2024-09-21T17:19:01.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:01 smithi157 ceph-mon[25926]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-21T17:19:04.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:03 smithi018 ceph-mon[23738]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:04.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:03 smithi157 ceph-mon[25926]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:06.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:05 smithi018 ceph-mon[23738]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:06.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:05 smithi157 ceph-mon[25926]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:07.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:07 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:07.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:07 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:07.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:07 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:07.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:07 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:08.031 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:08 smithi018 ceph-mon[23738]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:08.032 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:08.032 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:08.032 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:08.032 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:08 smithi157 ceph-mon[25926]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:10.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:10 smithi018 ceph-mon[23738]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:10.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:10 smithi157 ceph-mon[25926]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:10.736 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 3 on host 'smithi018' 2024-09-21T17:19:11.728 DEBUG:teuthology.orchestra.run.smithi018:osd.3> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T17:19:11.731 INFO:tasks.cephadm:Deploying osd.4 on smithi157 with /dev/vg_nvme/lv_4... 2024-09-21T17:19:11.731 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T17:19:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:11 smithi157 ceph-mon[25926]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:12.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:11 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:12.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:11 smithi157 ceph-mon[25926]: from='osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T17:19:12.070 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:11 smithi018 ceph-mon[23738]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:12.071 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:11 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:12.071 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:11 smithi018 ceph-mon[23738]: from='osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T17:19:12.999 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:12 smithi157 ceph-mon[25926]: from='osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T17:19:12.999 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:12 smithi157 ceph-mon[25926]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T17:19:12.999 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:12 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:19:13.000 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:12 smithi157 ceph-mon[25926]: from='osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T17:19:13.011 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:12 smithi018 ceph-mon[23738]: from='osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T17:19:13.011 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:12 smithi018 ceph-mon[23738]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T17:19:13.011 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:12 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:19:13.011 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:12 smithi018 ceph-mon[23738]: from='osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2024-09-21T17:19:13.011 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:19:12 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[44685]: 2024-09-21T17:19:12.741+0000 7f3f0c359700 -1 osd.3 0 waiting for initial osdmap 2024-09-21T17:19:13.012 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:19:12 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[44685]: 2024-09-21T17:19:12.745+0000 7f3f074f1700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:19:13.877 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:13 smithi018 ceph-mon[23738]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:13.877 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:13 smithi018 ceph-mon[23738]: from='osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-21T17:19:13.877 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:13 smithi018 ceph-mon[23738]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T17:19:13.877 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:19:13.877 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:13 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:19:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:13 smithi157 ceph-mon[25926]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:13 smithi157 ceph-mon[25926]: from='osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2024-09-21T17:19:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:13 smithi157 ceph-mon[25926]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T17:19:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:19:14.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:13 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:19:14.336 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:19:14.356 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi157:vg_nvme/lv_4 2024-09-21T17:19:14.934 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:14 smithi157 ceph-mon[25926]: purged_snaps scrub starts 2024-09-21T17:19:14.934 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:14 smithi157 ceph-mon[25926]: purged_snaps scrub ok 2024-09-21T17:19:14.935 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:14 smithi157 ceph-mon[25926]: osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783] boot 2024-09-21T17:19:14.935 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:14 smithi157 ceph-mon[25926]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T17:19:14.935 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:14 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:19:15.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:14 smithi018 ceph-mon[23738]: purged_snaps scrub starts 2024-09-21T17:19:15.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:14 smithi018 ceph-mon[23738]: purged_snaps scrub ok 2024-09-21T17:19:15.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:14 smithi018 ceph-mon[23738]: osd.3 [v2:172.21.15.18:6826/1384025783,v1:172.21.15.18:6827/1384025783] boot 2024-09-21T17:19:15.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:14 smithi018 ceph-mon[23738]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T17:19:15.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:14 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:19:15.985 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:15 smithi157 ceph-mon[25926]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:15.985 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:15 smithi157 ceph-mon[25926]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T17:19:15.985 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:15 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:16.013 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:15 smithi018 ceph-mon[23738]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T17:19:16.013 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:15 smithi018 ceph-mon[23738]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T17:19:16.013 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:15 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:16.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:16 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:19:16.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:16 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:19:16.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:16 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:16 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:19:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:16 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:19:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:16 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:17.933 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:17 smithi157 ceph-mon[25926]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi157:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:19:17.933 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:17 smithi157 ceph-mon[25926]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:18.063 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:17 smithi018 ceph-mon[23738]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi157:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:19:18.063 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:17 smithi018 ceph-mon[23738]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi018 to 3217M 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:19.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/1222228405' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef5e6e3f-e4ef-461b-b9df-21dde131c207"}]: dispatch 2024-09-21T17:19:19.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef5e6e3f-e4ef-461b-b9df-21dde131c207"}]: dispatch 2024-09-21T17:19:19.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef5e6e3f-e4ef-461b-b9df-21dde131c207"}]': finished 2024-09-21T17:19:19.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T17:19:19.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:19 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi018 to 3217M 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='client.? 172.21.15.157:0/1222228405' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef5e6e3f-e4ef-461b-b9df-21dde131c207"}]: dispatch 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ef5e6e3f-e4ef-461b-b9df-21dde131c207"}]: dispatch 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ef5e6e3f-e4ef-461b-b9df-21dde131c207"}]': finished 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T17:19:19.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:19 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:20.397 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:20 smithi018 ceph-mon[23738]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:20.397 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:20 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/3337800649' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:19:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:20 smithi157 ceph-mon[25926]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:20 smithi157 ceph-mon[25926]: from='client.? 172.21.15.157:0/3337800649' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:19:22.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:22 smithi018 ceph-mon[23738]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:22.495 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:22 smithi157 ceph-mon[25926]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:23.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:23.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:24 smithi157 ceph-mon[25926]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:24.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:24 smithi018 ceph-mon[23738]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:25.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T17:19:25.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:25.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T17:19:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:26.464 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:26 smithi157 ceph-mon[25926]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:26.464 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:26 smithi157 ceph-mon[25926]: Deploying daemon osd.4 on smithi157 2024-09-21T17:19:26.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:26 smithi018 ceph-mon[23738]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:26 smithi018 ceph-mon[23738]: Deploying daemon osd.4 on smithi157 2024-09-21T17:19:28.389 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:28 smithi157 ceph-mon[25926]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:28.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:28 smithi018 ceph-mon[23738]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:30 smithi157 ceph-mon[25926]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:30.541 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:30 smithi018 ceph-mon[23738]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:31 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:31 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:31 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:31 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:31 smithi157 ceph-mon[25926]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:31.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:31 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:31.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:31 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:31.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:31 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:31.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:31 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:31.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:31 smithi018 ceph-mon[23738]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:33 smithi157 ceph-mon[25926]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:33 smithi157 ceph-mon[25926]: from='osd.4 [v2:172.21.15.157:6800/2874067171,v1:172.21.15.157:6801/2874067171]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T17:19:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:33 smithi157 ceph-mon[25926]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T17:19:34.068 INFO:teuthology.orchestra.run.smithi157.stdout:Created osd(s) 4 on host 'smithi157' 2024-09-21T17:19:34.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:33 smithi018 ceph-mon[23738]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:34.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:33 smithi018 ceph-mon[23738]: from='osd.4 [v2:172.21.15.157:6800/2874067171,v1:172.21.15.157:6801/2874067171]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T17:19:34.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:33 smithi018 ceph-mon[23738]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T17:19:35.115 DEBUG:teuthology.orchestra.run.smithi157:osd.4> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T17:19:35.117 INFO:tasks.cephadm:Deploying osd.5 on smithi157 with /dev/vg_nvme/lv_3... 2024-09-21T17:19:35.117 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T17:19:35.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T17:19:35.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T17:19:35.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:35.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: from='osd.4 [v2:172.21.15.157:6800/2874067171,v1:172.21.15.157:6801/2874067171]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:19:35.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:19:35.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:35.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:19:35.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:19:35.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:35.179 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T17:19:35.179 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T17:19:35.180 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:35.180 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: from='osd.4 [v2:172.21.15.157:6800/2874067171,v1:172.21.15.157:6801/2874067171]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:19:35.180 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:19:35.180 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:35.180 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:19:35.180 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:19:35.180 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:36.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:35 smithi157 ceph-mon[25926]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:36.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:35 smithi157 ceph-mon[25926]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]': finished 2024-09-21T17:19:36.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:35 smithi157 ceph-mon[25926]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T17:19:36.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:36.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:36.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:35 smithi018 ceph-mon[23738]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:36.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:35 smithi018 ceph-mon[23738]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]': finished 2024-09-21T17:19:36.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:35 smithi018 ceph-mon[23738]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T17:19:36.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:36.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:37.275 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:36 smithi157 ceph-mon[25926]: purged_snaps scrub starts 2024-09-21T17:19:37.275 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:36 smithi157 ceph-mon[25926]: purged_snaps scrub ok 2024-09-21T17:19:37.275 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:36 smithi157 ceph-mon[25926]: osd.4 [v2:172.21.15.157:6800/2874067171,v1:172.21.15.157:6801/2874067171] boot 2024-09-21T17:19:37.275 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:36 smithi157 ceph-mon[25926]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T17:19:37.276 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:36 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:37.276 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:36 smithi157 ceph-mon[25926]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T17:19:37.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:36 smithi018 ceph-mon[23738]: purged_snaps scrub starts 2024-09-21T17:19:37.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:36 smithi018 ceph-mon[23738]: purged_snaps scrub ok 2024-09-21T17:19:37.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:36 smithi018 ceph-mon[23738]: osd.4 [v2:172.21.15.157:6800/2874067171,v1:172.21.15.157:6801/2874067171] boot 2024-09-21T17:19:37.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:36 smithi018 ceph-mon[23738]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T17:19:37.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:36 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:19:37.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:36 smithi018 ceph-mon[23738]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T17:19:38.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:37 smithi157 ceph-mon[25926]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:38.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:37 smithi157 ceph-mon[25926]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T17:19:38.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:37 smithi018 ceph-mon[23738]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T17:19:38.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:37 smithi018 ceph-mon[23738]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T17:19:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T17:19:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:40.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:40.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:40.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:40.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:40.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:40.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:40.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:40.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:40.658 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:19:40.678 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi157:vg_nvme/lv_3 2024-09-21T17:19:41.218 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:40 smithi157 ceph-mon[25926]: Detected new or changed devices on smithi157 2024-09-21T17:19:41.218 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:40 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi157 to 15936M 2024-09-21T17:19:41.218 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:40 smithi157 ceph-mon[25926]: osdmap e35: 5 total, 5 up, 5 in 2024-09-21T17:19:41.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:40 smithi018 ceph-mon[23738]: Detected new or changed devices on smithi157 2024-09-21T17:19:41.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:40 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi157 to 15936M 2024-09-21T17:19:41.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:40 smithi018 ceph-mon[23738]: osdmap e35: 5 total, 5 up, 5 in 2024-09-21T17:19:42.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:41 smithi157 ceph-mon[25926]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:42.381 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:41 smithi018 ceph-mon[23738]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:43.141 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:19:43.141 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:19:43.142 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:19:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:19:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:44.169 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:43 smithi157 ceph-mon[25926]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-09-21T17:19:44.169 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:43 smithi157 ceph-mon[25926]: from='client.24163 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi157:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:19:44.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:43 smithi018 ceph-mon[23738]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-09-21T17:19:44.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:43 smithi018 ceph-mon[23738]: from='client.24163 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi157:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:19:46.234 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:45 smithi157 ceph-mon[25926]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-21T17:19:46.234 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:45 smithi157 ceph-mon[25926]: from='client.? 172.21.15.157:0/2037944649' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c046bd-29fd-46c6-affd-f7f1c21c8079"}]: dispatch 2024-09-21T17:19:46.235 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:45 smithi157 ceph-mon[25926]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c046bd-29fd-46c6-affd-f7f1c21c8079"}]: dispatch 2024-09-21T17:19:46.235 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:45 smithi157 ceph-mon[25926]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0c046bd-29fd-46c6-affd-f7f1c21c8079"}]': finished 2024-09-21T17:19:46.235 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:45 smithi157 ceph-mon[25926]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T17:19:46.235 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:45 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:19:46.235 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:45 smithi157 ceph-mon[25926]: from='client.? 172.21.15.157:0/1492156865' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:19:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:45 smithi018 ceph-mon[23738]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-21T17:19:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:45 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/2037944649' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c046bd-29fd-46c6-affd-f7f1c21c8079"}]: dispatch 2024-09-21T17:19:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:45 smithi018 ceph-mon[23738]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c046bd-29fd-46c6-affd-f7f1c21c8079"}]: dispatch 2024-09-21T17:19:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:45 smithi018 ceph-mon[23738]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0c046bd-29fd-46c6-affd-f7f1c21c8079"}]': finished 2024-09-21T17:19:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:45 smithi018 ceph-mon[23738]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T17:19:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:45 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:19:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:45 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/1492156865' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:19:48.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:47 smithi157 ceph-mon[25926]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-21T17:19:48.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:47 smithi018 ceph-mon[23738]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-21T17:19:50.159 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:49 smithi157 ceph-mon[25926]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-21T17:19:50.294 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:49 smithi018 ceph-mon[23738]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-09-21T17:19:52.227 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:51 smithi157 ceph-mon[25926]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T17:19:52.227 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T17:19:52.227 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:51 smithi018 ceph-mon[23738]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T17:19:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T17:19:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:52 smithi157 ceph-mon[25926]: Deploying daemon osd.5 on smithi157 2024-09-21T17:19:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:52 smithi018 ceph-mon[23738]: Deploying daemon osd.5 on smithi157 2024-09-21T17:19:54.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:53 smithi157 ceph-mon[25926]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:54.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:53 smithi018 ceph-mon[23738]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:56.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:55 smithi157 ceph-mon[25926]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:56.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:55 smithi018 ceph-mon[23738]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:58 smithi018 ceph-mon[23738]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:58 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:58 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:58 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:19:58 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:19:58.504 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:58 smithi157 ceph-mon[25926]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:19:58.504 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:58 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:19:58.504 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:58 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:19:58.504 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:58 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:19:58.505 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:19:58 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:00.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:00 smithi157 ceph-mon[25926]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:20:00.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:00 smithi157 ceph-mon[25926]: overall HEALTH_OK 2024-09-21T17:20:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:00 smithi018 ceph-mon[23738]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:20:00.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:00 smithi018 ceph-mon[23738]: overall HEALTH_OK 2024-09-21T17:20:00.666 INFO:teuthology.orchestra.run.smithi157.stdout:Created osd(s) 5 on host 'smithi157' 2024-09-21T17:20:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:01 smithi157 ceph-mon[25926]: from='osd.5 [v2:172.21.15.157:6808/3847204715,v1:172.21.15.157:6809/3847204715]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T17:20:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:01 smithi157 ceph-mon[25926]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T17:20:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:01 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:01.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:01 smithi018 ceph-mon[23738]: from='osd.5 [v2:172.21.15.157:6808/3847204715,v1:172.21.15.157:6809/3847204715]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T17:20:01.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:01 smithi018 ceph-mon[23738]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T17:20:01.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:01 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:01.733 DEBUG:teuthology.orchestra.run.smithi157:osd.5> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T17:20:01.735 INFO:tasks.cephadm:Deploying osd.6 on smithi157 with /dev/vg_nvme/lv_2... 2024-09-21T17:20:01.735 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T17:20:02.352 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-mon[25926]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:20:02.352 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-mon[25926]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T17:20:02.352 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-mon[25926]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T17:20:02.352 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:02.352 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-mon[25926]: from='osd.5 [v2:172.21.15.157:6808/3847204715,v1:172.21.15.157:6809/3847204715]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:02.352 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-mon[25926]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:02.352 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:02.353 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[32997]: 2024-09-21T17:20:02.099+0000 7f14af040700 -1 osd.5 0 waiting for initial osdmap 2024-09-21T17:20:02.353 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:20:02 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[32997]: 2024-09-21T17:20:02.104+0000 7f14a99d7700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:20:02.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:02 smithi018 ceph-mon[23738]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:20:02.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:02 smithi018 ceph-mon[23738]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T17:20:02.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:02 smithi018 ceph-mon[23738]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T17:20:02.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:02 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:02.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:02 smithi018 ceph-mon[23738]: from='osd.5 [v2:172.21.15.157:6808/3847204715,v1:172.21.15.157:6809/3847204715]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:02.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:02 smithi018 ceph-mon[23738]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:02.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:02 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:03.399 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:03 smithi157 ceph-mon[25926]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]': finished 2024-09-21T17:20:03.400 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:03 smithi157 ceph-mon[25926]: osdmap e38: 6 total, 5 up, 6 in 2024-09-21T17:20:03.400 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:03 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:03.400 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:03 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:03.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:03 smithi018 ceph-mon[23738]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]': finished 2024-09-21T17:20:03.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:03 smithi018 ceph-mon[23738]: osdmap e38: 6 total, 5 up, 6 in 2024-09-21T17:20:03.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:03 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:03.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:03 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:04.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:04 smithi157 ceph-mon[25926]: purged_snaps scrub starts 2024-09-21T17:20:04.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:04 smithi157 ceph-mon[25926]: purged_snaps scrub ok 2024-09-21T17:20:04.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:04 smithi157 ceph-mon[25926]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:20:04.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:04 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:04.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:04 smithi157 ceph-mon[25926]: osd.5 [v2:172.21.15.157:6808/3847204715,v1:172.21.15.157:6809/3847204715] boot 2024-09-21T17:20:04.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:04 smithi157 ceph-mon[25926]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T17:20:04.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:04 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:04 smithi018 ceph-mon[23738]: purged_snaps scrub starts 2024-09-21T17:20:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:04 smithi018 ceph-mon[23738]: purged_snaps scrub ok 2024-09-21T17:20:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:04 smithi018 ceph-mon[23738]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T17:20:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:04 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:04 smithi018 ceph-mon[23738]: osd.5 [v2:172.21.15.157:6808/3847204715,v1:172.21.15.157:6809/3847204715] boot 2024-09-21T17:20:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:04 smithi018 ceph-mon[23738]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T17:20:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:04 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:20:05.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:05 smithi018 ceph-mon[23738]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T17:20:05.527 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:05 smithi157 ceph-mon[25926]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T17:20:05.532 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:20:05.553 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi157:vg_nvme/lv_2 2024-09-21T17:20:06.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:06 smithi018 ceph-mon[23738]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:06.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:06 smithi018 ceph-mon[23738]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T17:20:06.492 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:06 smithi157 ceph-mon[25926]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:06.492 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:06 smithi157 ceph-mon[25926]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T17:20:07.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:07 smithi018 ceph-mon[23738]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T17:20:07.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:07 smithi018 ceph-mon[23738]: osdmap e43: 6 total, 6 up, 6 in 2024-09-21T17:20:07.455 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:07 smithi157 ceph-mon[25926]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T17:20:07.455 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:07 smithi157 ceph-mon[25926]: osdmap e43: 6 total, 6 up, 6 in 2024-09-21T17:20:08.200 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: pgmap v136: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:08.201 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:08 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:08.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: pgmap v136: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:08.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:08.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:08.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:08.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:08.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:08.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:08.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:08.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:08.404 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:08 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:09 smithi157 ceph-mon[25926]: Detected new or changed devices on smithi157 2024-09-21T17:20:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:09 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi157 to 7968M 2024-09-21T17:20:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:09 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:20:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:09 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:20:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:09 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:09.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:09 smithi018 ceph-mon[23738]: Detected new or changed devices on smithi157 2024-09-21T17:20:09.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:09 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi157 to 7968M 2024-09-21T17:20:09.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:09 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:20:09.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:09 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:20:09.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:09 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:10.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:10 smithi157 ceph-mon[25926]: pgmap v138: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:10.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:10 smithi157 ceph-mon[25926]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi157:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:20:10.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:10 smithi018 ceph-mon[23738]: pgmap v138: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:10.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:10 smithi018 ceph-mon[23738]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi157:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:20:11.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:11 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/73426755' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8e90df-1b3e-4b65-8d15-d0beab02c1e9"}]: dispatch 2024-09-21T17:20:11.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:11 smithi018 ceph-mon[23738]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8e90df-1b3e-4b65-8d15-d0beab02c1e9"}]: dispatch 2024-09-21T17:20:11.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:11 smithi018 ceph-mon[23738]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc8e90df-1b3e-4b65-8d15-d0beab02c1e9"}]': finished 2024-09-21T17:20:11.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:11 smithi018 ceph-mon[23738]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T17:20:11.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:11 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:11 smithi157 ceph-mon[25926]: from='client.? 172.21.15.157:0/73426755' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8e90df-1b3e-4b65-8d15-d0beab02c1e9"}]: dispatch 2024-09-21T17:20:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:11 smithi157 ceph-mon[25926]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc8e90df-1b3e-4b65-8d15-d0beab02c1e9"}]: dispatch 2024-09-21T17:20:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:11 smithi157 ceph-mon[25926]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc8e90df-1b3e-4b65-8d15-d0beab02c1e9"}]': finished 2024-09-21T17:20:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:11 smithi157 ceph-mon[25926]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T17:20:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:11 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:12.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:12 smithi018 ceph-mon[23738]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:12.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:12 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/936904395' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:20:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:12 smithi157 ceph-mon[25926]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:12 smithi157 ceph-mon[25926]: from='client.? 172.21.15.157:0/936904395' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:20:14.391 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:14 smithi157 ceph-mon[25926]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:14.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:14 smithi018 ceph-mon[23738]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:16.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:16 smithi018 ceph-mon[23738]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:16 smithi157 ceph-mon[25926]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:18.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:18 smithi018 ceph-mon[23738]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:18.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:18 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T17:20:18.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:18 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:18.469 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:18 smithi157 ceph-mon[25926]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:18.469 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:18 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T17:20:18.469 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:18 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:19.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:19 smithi157 ceph-mon[25926]: Deploying daemon osd.6 on smithi157 2024-09-21T17:20:19.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:19 smithi018 ceph-mon[23738]: Deploying daemon osd.6 on smithi157 2024-09-21T17:20:20.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:20 smithi018 ceph-mon[23738]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:20.483 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:20 smithi157 ceph-mon[25926]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:22.349 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:22 smithi157 ceph-mon[25926]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:22.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:22 smithi018 ceph-mon[23738]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:23.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:23.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:23.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:23.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:23.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:23 smithi157 ceph-mon[25926]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:23.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:23.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:23.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:23.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:23.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:23 smithi018 ceph-mon[23738]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:25.579 INFO:teuthology.orchestra.run.smithi157.stdout:Created osd(s) 6 on host 'smithi157' 2024-09-21T17:20:25.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:25 smithi018 ceph-mon[23738]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:25.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:25.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:25 smithi018 ceph-mon[23738]: from='osd.6 [v2:172.21.15.157:6816/1541443444,v1:172.21.15.157:6817/1541443444]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T17:20:25.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:25 smithi018 ceph-mon[23738]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T17:20:25.921 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:25 smithi157 ceph-mon[25926]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:25.921 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:25.921 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:25 smithi157 ceph-mon[25926]: from='osd.6 [v2:172.21.15.157:6816/1541443444,v1:172.21.15.157:6817/1541443444]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T17:20:25.922 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:25 smithi157 ceph-mon[25926]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T17:20:26.262 DEBUG:teuthology.orchestra.run.smithi157:osd.6> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.6.service 2024-09-21T17:20:26.264 INFO:tasks.cephadm:Deploying osd.7 on smithi157 with /dev/vg_nvme/lv_1... 2024-09-21T17:20:26.264 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T17:20:27.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:27 smithi018 ceph-mon[23738]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:27.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:27 smithi018 ceph-mon[23738]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T17:20:27.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:27 smithi018 ceph-mon[23738]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T17:20:27.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:27 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:27.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:27 smithi018 ceph-mon[23738]: from='osd.6 [v2:172.21.15.157:6816/1541443444,v1:172.21.15.157:6817/1541443444]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:27.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:27 smithi018 ceph-mon[23738]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:28.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:27 smithi157 ceph-mon[25926]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:28.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:27 smithi157 ceph-mon[25926]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T17:20:28.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:27 smithi157 ceph-mon[25926]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T17:20:28.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:27 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:28.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:27 smithi157 ceph-mon[25926]: from='osd.6 [v2:172.21.15.157:6816/1541443444,v1:172.21.15.157:6817/1541443444]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:28.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:27 smithi157 ceph-mon[25926]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:28.029 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:20:27 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[35974]: 2024-09-21T17:20:27.583+0000 7f9ef6143700 -1 osd.6 0 waiting for initial osdmap 2024-09-21T17:20:28.029 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:20:27 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[35974]: 2024-09-21T17:20:27.589+0000 7f9ef12db700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:20:28.729 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:28 smithi157 ceph-mon[25926]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]': finished 2024-09-21T17:20:28.729 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:28 smithi157 ceph-mon[25926]: osdmap e46: 7 total, 6 up, 7 in 2024-09-21T17:20:28.729 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:28 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:28.729 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:28 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:28.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:28 smithi018 ceph-mon[23738]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]': finished 2024-09-21T17:20:28.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:28 smithi018 ceph-mon[23738]: osdmap e46: 7 total, 6 up, 7 in 2024-09-21T17:20:28.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:28 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:28.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:28 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:29.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:29 smithi157 ceph-mon[25926]: purged_snaps scrub starts 2024-09-21T17:20:29.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:29 smithi157 ceph-mon[25926]: purged_snaps scrub ok 2024-09-21T17:20:29.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:29 smithi157 ceph-mon[25926]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:29.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:29 smithi157 ceph-mon[25926]: osd.6 [v2:172.21.15.157:6816/1541443444,v1:172.21.15.157:6817/1541443444] boot 2024-09-21T17:20:29.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:29 smithi157 ceph-mon[25926]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T17:20:29.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:29 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:29.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:29 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:29.787 INFO:teuthology.orchestra.run.smithi157.stdout: 2024-09-21T17:20:29.805 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi157:vg_nvme/lv_1 2024-09-21T17:20:29.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:29 smithi018 ceph-mon[23738]: purged_snaps scrub starts 2024-09-21T17:20:29.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:29 smithi018 ceph-mon[23738]: purged_snaps scrub ok 2024-09-21T17:20:29.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:29 smithi018 ceph-mon[23738]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T17:20:29.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:29 smithi018 ceph-mon[23738]: osd.6 [v2:172.21.15.157:6816/1541443444,v1:172.21.15.157:6817/1541443444] boot 2024-09-21T17:20:29.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:29 smithi018 ceph-mon[23738]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T17:20:29.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:29 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:20:29.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:29 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:30.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:30 smithi018 ceph-mon[23738]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T17:20:30.923 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:30 smithi157 ceph-mon[25926]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T17:20:31.877 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:31 smithi157 ceph-mon[25926]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:31.877 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:31 smithi157 ceph-mon[25926]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T17:20:31.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:31 smithi018 ceph-mon[23738]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:31.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:31 smithi018 ceph-mon[23738]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T17:20:32.724 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:32 smithi157 ceph-mon[25926]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T17:20:32.725 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:20:32.725 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:20:32.725 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:32.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:32 smithi018 ceph-mon[23738]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T17:20:32.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T17:20:32.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T17:20:32.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:33.756 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:33 smithi157 ceph-mon[25926]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi157:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:20:33.756 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:33 smithi157 ceph-mon[25926]: pgmap v157: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-21T17:20:33.757 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:33 smithi157 ceph-mon[25926]: osdmap e51: 7 total, 7 up, 7 in 2024-09-21T17:20:33.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:33 smithi018 ceph-mon[23738]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi157:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:20:33.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:33 smithi018 ceph-mon[23738]: pgmap v157: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-21T17:20:33.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:33 smithi018 ceph-mon[23738]: osdmap e51: 7 total, 7 up, 7 in 2024-09-21T17:20:34.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:20:34.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:20:35.005 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:20:35.005 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: Detected new or changed devices on smithi157 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi157 to 5312M 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:35.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:35.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:35.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:35.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:36.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-21T17:20:36.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: Detected new or changed devices on smithi157 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi157 to 5312M 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:36.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:36.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:36 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/2163016918' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "495f1013-7888-40e2-8e13-e8568849277b"}]: dispatch 2024-09-21T17:20:36.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:36 smithi018 ceph-mon[23738]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "495f1013-7888-40e2-8e13-e8568849277b"}]: dispatch 2024-09-21T17:20:36.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:36 smithi018 ceph-mon[23738]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "495f1013-7888-40e2-8e13-e8568849277b"}]': finished 2024-09-21T17:20:36.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:36 smithi018 ceph-mon[23738]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T17:20:36.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:36 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:36 smithi157 ceph-mon[25926]: from='client.? 172.21.15.157:0/2163016918' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "495f1013-7888-40e2-8e13-e8568849277b"}]: dispatch 2024-09-21T17:20:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:36 smithi157 ceph-mon[25926]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "495f1013-7888-40e2-8e13-e8568849277b"}]: dispatch 2024-09-21T17:20:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:36 smithi157 ceph-mon[25926]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "495f1013-7888-40e2-8e13-e8568849277b"}]': finished 2024-09-21T17:20:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:36 smithi157 ceph-mon[25926]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T17:20:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:36 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:37.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:37 smithi018 ceph-mon[23738]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-21T17:20:37.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:37 smithi018 ceph-mon[23738]: from='client.? 172.21.15.157:0/693926403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:20:38.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:37 smithi157 ceph-mon[25926]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-21T17:20:38.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:37 smithi157 ceph-mon[25926]: from='client.? 172.21.15.157:0/693926403' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T17:20:39.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:39 smithi157 ceph-mon[25926]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:39.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:39 smithi018 ceph-mon[23738]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:41.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:41 smithi018 ceph-mon[23738]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:42.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:41 smithi157 ceph-mon[25926]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:43.131 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T17:20:43.131 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:43.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T17:20:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:44.231 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:43 smithi157 ceph-mon[25926]: Deploying daemon osd.7 on smithi157 2024-09-21T17:20:44.231 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:43 smithi157 ceph-mon[25926]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:44.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:43 smithi018 ceph-mon[23738]: Deploying daemon osd.7 on smithi157 2024-09-21T17:20:44.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:43 smithi018 ceph-mon[23738]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:46.220 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:45 smithi157 ceph-mon[25926]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:46.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:45 smithi018 ceph-mon[23738]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:48.238 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:47 smithi157 ceph-mon[25926]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:48.238 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:47 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:48.239 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:47 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:48.239 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:47 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:48.239 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:47 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:48.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:47 smithi018 ceph-mon[23738]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:47 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:47 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:47 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:47 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:50.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:49 smithi157 ceph-mon[25926]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:50.294 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:49 smithi018 ceph-mon[23738]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:51 smithi157 ceph-mon[25926]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:51 smithi157 ceph-mon[25926]: from='osd.7 [v2:172.21.15.157:6824/1991105348,v1:172.21.15.157:6825/1991105348]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T17:20:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:51 smithi157 ceph-mon[25926]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T17:20:52.096 INFO:teuthology.orchestra.run.smithi157.stdout:Created osd(s) 7 on host 'smithi157' 2024-09-21T17:20:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:51 smithi018 ceph-mon[23738]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:51 smithi018 ceph-mon[23738]: from='osd.7 [v2:172.21.15.157:6824/1991105348,v1:172.21.15.157:6825/1991105348]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T17:20:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:51 smithi018 ceph-mon[23738]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T17:20:52.776 DEBUG:teuthology.orchestra.run.smithi157:osd.7> sudo journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.7.service 2024-09-21T17:20:52.779 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T17:20:52.779 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T17:20:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:52 smithi157 ceph-mon[25926]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T17:20:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:52 smithi157 ceph-mon[25926]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T17:20:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:52 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:52 smithi157 ceph-mon[25926]: from='osd.7 [v2:172.21.15.157:6824/1991105348,v1:172.21.15.157:6825/1991105348]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:52 smithi157 ceph-mon[25926]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:52 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:53.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:20:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[39009]: 2024-09-21T17:20:52.928+0000 7f3f56aae700 -1 osd.7 0 waiting for initial osdmap 2024-09-21T17:20:53.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:20:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[39009]: 2024-09-21T17:20:52.936+0000 7f3f52c48700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:20:53.110 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:52 smithi018 ceph-mon[23738]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T17:20:53.110 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:52 smithi018 ceph-mon[23738]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T17:20:53.110 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:52 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:53.110 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:52 smithi018 ceph-mon[23738]: from='osd.7 [v2:172.21.15.157:6824/1991105348,v1:172.21.15.157:6825/1991105348]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:53.110 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:52 smithi018 ceph-mon[23738]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]: dispatch 2024-09-21T17:20:53.110 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:52 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:54.196 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:53 smithi018 ceph-mon[23738]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:54.196 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:53 smithi018 ceph-mon[23738]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]': finished 2024-09-21T17:20:54.196 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:53 smithi018 ceph-mon[23738]: osdmap e54: 8 total, 7 up, 8 in 2024-09-21T17:20:54.196 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:53 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:54.196 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:53 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:54.196 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:53 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:54.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:53 smithi157 ceph-mon[25926]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T17:20:54.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:53 smithi157 ceph-mon[25926]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]}]': finished 2024-09-21T17:20:54.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:53 smithi157 ceph-mon[25926]: osdmap e54: 8 total, 7 up, 8 in 2024-09-21T17:20:54.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:53 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:54.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:53 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:54.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:53 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:54.918 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:20:55.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:54 smithi018 ceph-mon[23738]: purged_snaps scrub starts 2024-09-21T17:20:55.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:54 smithi018 ceph-mon[23738]: purged_snaps scrub ok 2024-09-21T17:20:55.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:54 smithi018 ceph-mon[23738]: osd.7 [v2:172.21.15.157:6824/1991105348,v1:172.21.15.157:6825/1991105348] boot 2024-09-21T17:20:55.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:54 smithi018 ceph-mon[23738]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T17:20:55.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:55.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:54 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/239075909' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T17:20:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:54 smithi157 ceph-mon[25926]: purged_snaps scrub starts 2024-09-21T17:20:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:54 smithi157 ceph-mon[25926]: purged_snaps scrub ok 2024-09-21T17:20:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:54 smithi157 ceph-mon[25926]: osd.7 [v2:172.21.15.157:6824/1991105348,v1:172.21.15.157:6825/1991105348] boot 2024-09-21T17:20:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:54 smithi157 ceph-mon[25926]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T17:20:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:20:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:54 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/239075909' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T17:20:55.655 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726939253,"num_in_osds":8,"osd_in_since":1726939236,"num_remapped_pgs":1} 2024-09-21T17:20:55.655 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T17:20:55.883 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:20:56.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:55 smithi157 ceph-mon[25926]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:20:56.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:55 smithi157 ceph-mon[25926]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T17:20:56.247 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:55 smithi018 ceph-mon[23738]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:20:56.247 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:55 smithi018 ceph-mon[23738]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: Detected new or changed devices on smithi157 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi157 to 3984M 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:57.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:57.271 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:57.271 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:57.271 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:57.271 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:57.271 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:56 smithi018 ceph-mon[23738]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: Detected new or changed devices on smithi157 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi157 to 3984M 2024-09-21T17:20:57.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:57.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:57.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:20:57.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:20:57.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:20:57.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:20:57.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:56 smithi157 ceph-mon[25926]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T17:20:57.824 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:20:57.824 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":58,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","created":"2024-09-21T17:14:47.334351+0000","modified":"2024-09-21T17:20:56.901563+0000","last_up_change":"2024-09-21T17:20:53.920499+0000","last_in_change":"2024-09-21T17:20:36.054112+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T17:18:45.020287+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"ac9a72bf-1285-4f67-b0a4-0e39841eae95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6803","nonce":2412849795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6805","nonce":2412849795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6809","nonce":2412849795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6807","nonce":2412849795}]},"public_addr":"172.21.15.18:6803/2412849795","cluster_addr":"172.21.15.18:6805/2412849795","heartbeat_back_addr":"172.21.15.18:6809/2412849795","heartbeat_front_addr":"172.21.15.18:6807/2412849795","state":["exists","up"]},{"osd":1,"uuid":"bd92a0a6-7111-4ed3-b70b-20c6dd0a45df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6811","nonce":3829486049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6813","nonce":3829486049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6817","nonce":3829486049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6815","nonce":3829486049}]},"public_addr":"172.21.15.18:6811/3829486049","cluster_addr":"172.21.15.18:6813/3829486049","heartbeat_back_addr":"172.21.15.18:6817/3829486049","heartbeat_front_addr":"172.21.15.18:6815/3829486049","state":["exists","up"]},{"osd":2,"uuid":"40e5c057-8087-4fbf-9d48-37a163accaad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6819","nonce":2209132202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6821","nonce":2209132202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6825","nonce":2209132202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6823","nonce":2209132202}]},"public_addr":"172.21.15.18:6819/2209132202","cluster_addr":"172.21.15.18:6821/2209132202","heartbeat_back_addr":"172.21.15.18:6825/2209132202","heartbeat_front_addr":"172.21.15.18:6823/2209132202","state":["exists","up"]},{"osd":3,"uuid":"8be04aa4-bfcc-4407-bb16-8500a8f5ae66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6827","nonce":1384025783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6829","nonce":1384025783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6833","nonce":1384025783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6831","nonce":1384025783}]},"public_addr":"172.21.15.18:6827/1384025783","cluster_addr":"172.21.15.18:6829/1384025783","heartbeat_back_addr":"172.21.15.18:6833/1384025783","heartbeat_front_addr":"172.21.15.18:6831/1384025783","state":["exists","up"]},{"osd":4,"uuid":"ef5e6e3f-e4ef-461b-b9df-21dde131c207","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6800","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6801","nonce":2874067171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6802","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6803","nonce":2874067171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6806","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6807","nonce":2874067171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6804","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6805","nonce":2874067171}]},"public_addr":"172.21.15.157:6801/2874067171","cluster_addr":"172.21.15.157:6803/2874067171","heartbeat_back_addr":"172.21.15.157:6807/2874067171","heartbeat_front_addr":"172.21.15.157:6805/2874067171","state":["exists","up"]},{"osd":5,"uuid":"a0c046bd-29fd-46c6-affd-f7f1c21c8079","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6808","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6809","nonce":3847204715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6810","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6811","nonce":3847204715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6814","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6815","nonce":3847204715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6812","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6813","nonce":3847204715}]},"public_addr":"172.21.15.157:6809/3847204715","cluster_addr":"172.21.15.157:6811/3847204715","heartbeat_back_addr":"172.21.15.157:6815/3847204715","heartbeat_front_addr":"172.21.15.157:6813/3847204715","state":["exists","up"]},{"osd":6,"uuid":"dc8e90df-1b3e-4b65-8d15-d0beab02c1e9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6816","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6817","nonce":1541443444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6818","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6819","nonce":1541443444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6822","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6823","nonce":1541443444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6820","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6821","nonce":1541443444}]},"public_addr":"172.21.15.157:6817/1541443444","cluster_addr":"172.21.15.157:6819/1541443444","heartbeat_back_addr":"172.21.15.157:6823/1541443444","heartbeat_front_addr":"172.21.15.157:6821/1541443444","state":["exists","up"]},{"osd":7,"uuid":"495f1013-7888-40e2-8e13-e8568849277b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6824","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6825","nonce":1991105348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6826","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6827","nonce":1991105348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6830","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6831","nonce":1991105348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6828","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6829","nonce":1991105348}]},"public_addr":"172.21.15.157:6825/1991105348","cluster_addr":"172.21.15.157:6827/1991105348","heartbeat_back_addr":"172.21.15.157:6831/1991105348","heartbeat_front_addr":"172.21.15.157:6829/1991105348","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:17:43.350412+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:18:14.269874+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:18:42.389028+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:19:11.897596+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:19:34.298711+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:01.188489+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:26.634387+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:51.945628+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:6801/1900679499":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/307040823":"2024-09-22T17:16:34.462248+0000","172.21.15.18:6800/1900679499":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/3338538266":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/1997413017":"2024-09-22T17:15:42.080516+0000","172.21.15.18:6801/2012086580":"2024-09-22T17:15:42.080516+0000","172.21.15.18:6800/2012086580":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/2286457948":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/3115032843":"2024-09-22T17:15:11.179851+0000","172.21.15.18:6800/902384568":"2024-09-22T17:15:11.179851+0000","172.21.15.18:0/2764100546":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/547361119":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/842545281":"2024-09-22T17:15:11.179851+0000","172.21.15.18:0/3015432587":"2024-09-22T17:15:11.179851+0000","172.21.15.18:6801/902384568":"2024-09-22T17:15:11.179851+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T17:20:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:58 smithi018 ceph-mon[23738]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:20:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:58 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2796503100' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T17:20:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:20:58 smithi018 ceph-mon[23738]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T17:20:58.521 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T17:18:45.020287+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-21T17:20:58.521 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T17:20:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:58 smithi157 ceph-mon[25926]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:20:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:58 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2796503100' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T17:20:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:20:58 smithi157 ceph-mon[25926]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T17:20:58.743 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:00.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:00 smithi018 ceph-mon[23738]: pgmap v179: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:00 smithi157 ceph-mon[25926]: pgmap v179: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:00.567 INFO:teuthology.orchestra.run.smithi018.stdout:pg_num: 1 2024-09-21T17:21:01.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:01 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1679202134' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T17:21:01.293 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T17:21:01.294 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T17:21:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:01 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/1679202134' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T17:21:02.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:02 smithi018 ceph-mon[23738]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-09-21T17:21:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:02 smithi157 ceph-mon[25926]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-09-21T17:21:03.297 INFO:teuthology.orchestra.run.smithi018.stdout:[client.0] 2024-09-21T17:21:03.297 INFO:teuthology.orchestra.run.smithi018.stdout: key = AQB/AO9mkExnERAA1CXoobDEpwy2LRE6dmO/9w== 2024-09-21T17:21:03.999 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2024-09-21T17:21:04.000 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T17:21:04.000 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T17:21:04.042 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T17:21:04.300 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:04.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:04 smithi018 ceph-mon[23738]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-21T17:21:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:04 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2452208418' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T17:21:04.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:04 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2452208418' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T17:21:04.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:04 smithi157 ceph-mon[25926]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-21T17:21:04.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:04 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2452208418' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T17:21:04.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:04 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2452208418' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T17:21:06.293 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:21:06.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:06 smithi018 ceph-mon[23738]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T17:21:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:06 smithi157 ceph-mon[25926]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T17:21:06.948 INFO:teuthology.orchestra.run.smithi018.stdout:[{"version":20,"timestamp":"2024-09-21T17:20:56.092837+0000","name":"","changes":[{"name":"osd/host:smithi157/osd_memory_target","previous_value":"5570077900","new_value":"4177558425"}]}] 2024-09-21T17:21:06.948 INFO:tasks.ceph_manager:config epoch is 20 2024-09-21T17:21:06.948 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T17:21:06.948 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T17:21:06.949 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T17:21:07.172 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:07.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:07 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/659222754' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T17:21:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:07 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/659222754' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T17:21:08.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:08 smithi018 ceph-mon[23738]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-21T17:21:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:08 smithi157 ceph-mon[25926]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-21T17:21:09.068 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:21:09.390 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:09 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/343139742' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T17:21:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:09 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/343139742' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T17:21:09.737 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi018.jndyyn","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6800","nonce":2928961376},{"type":"v1","addr":"172.21.15.18:6801","nonce":2928961376}]},"active_addr":"172.21.15.18:6801/2928961376","active_change":"2024-09-21T17:16:34.462528+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi157.oowooq","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":2481763930}]},{"addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":2720459882}]},{"addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":290992792}]},{"addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":4279991965}]}]}} 2024-09-21T17:21:09.740 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T17:21:09.740 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T17:21:09.741 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T17:21:09.964 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:10.336 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:10 smithi018 ceph-mon[23738]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T17:21:10.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:10 smithi157 ceph-mon[25926]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T17:21:11.906 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:21:11.907 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":59,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","created":"2024-09-21T17:14:47.334351+0000","modified":"2024-09-21T17:20:57.906341+0000","last_up_change":"2024-09-21T17:20:53.920499+0000","last_in_change":"2024-09-21T17:20:36.054112+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T17:18:45.020287+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"ac9a72bf-1285-4f67-b0a4-0e39841eae95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6803","nonce":2412849795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6805","nonce":2412849795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6809","nonce":2412849795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6807","nonce":2412849795}]},"public_addr":"172.21.15.18:6803/2412849795","cluster_addr":"172.21.15.18:6805/2412849795","heartbeat_back_addr":"172.21.15.18:6809/2412849795","heartbeat_front_addr":"172.21.15.18:6807/2412849795","state":["exists","up"]},{"osd":1,"uuid":"bd92a0a6-7111-4ed3-b70b-20c6dd0a45df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6811","nonce":3829486049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6813","nonce":3829486049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6817","nonce":3829486049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6815","nonce":3829486049}]},"public_addr":"172.21.15.18:6811/3829486049","cluster_addr":"172.21.15.18:6813/3829486049","heartbeat_back_addr":"172.21.15.18:6817/3829486049","heartbeat_front_addr":"172.21.15.18:6815/3829486049","state":["exists","up"]},{"osd":2,"uuid":"40e5c057-8087-4fbf-9d48-37a163accaad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6819","nonce":2209132202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6821","nonce":2209132202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6825","nonce":2209132202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6823","nonce":2209132202}]},"public_addr":"172.21.15.18:6819/2209132202","cluster_addr":"172.21.15.18:6821/2209132202","heartbeat_back_addr":"172.21.15.18:6825/2209132202","heartbeat_front_addr":"172.21.15.18:6823/2209132202","state":["exists","up"]},{"osd":3,"uuid":"8be04aa4-bfcc-4407-bb16-8500a8f5ae66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6827","nonce":1384025783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6829","nonce":1384025783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6833","nonce":1384025783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6831","nonce":1384025783}]},"public_addr":"172.21.15.18:6827/1384025783","cluster_addr":"172.21.15.18:6829/1384025783","heartbeat_back_addr":"172.21.15.18:6833/1384025783","heartbeat_front_addr":"172.21.15.18:6831/1384025783","state":["exists","up"]},{"osd":4,"uuid":"ef5e6e3f-e4ef-461b-b9df-21dde131c207","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6800","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6801","nonce":2874067171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6802","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6803","nonce":2874067171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6806","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6807","nonce":2874067171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6804","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6805","nonce":2874067171}]},"public_addr":"172.21.15.157:6801/2874067171","cluster_addr":"172.21.15.157:6803/2874067171","heartbeat_back_addr":"172.21.15.157:6807/2874067171","heartbeat_front_addr":"172.21.15.157:6805/2874067171","state":["exists","up"]},{"osd":5,"uuid":"a0c046bd-29fd-46c6-affd-f7f1c21c8079","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6808","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6809","nonce":3847204715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6810","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6811","nonce":3847204715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6814","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6815","nonce":3847204715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6812","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6813","nonce":3847204715}]},"public_addr":"172.21.15.157:6809/3847204715","cluster_addr":"172.21.15.157:6811/3847204715","heartbeat_back_addr":"172.21.15.157:6815/3847204715","heartbeat_front_addr":"172.21.15.157:6813/3847204715","state":["exists","up"]},{"osd":6,"uuid":"dc8e90df-1b3e-4b65-8d15-d0beab02c1e9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6816","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6817","nonce":1541443444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6818","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6819","nonce":1541443444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6822","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6823","nonce":1541443444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6820","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6821","nonce":1541443444}]},"public_addr":"172.21.15.157:6817/1541443444","cluster_addr":"172.21.15.157:6819/1541443444","heartbeat_back_addr":"172.21.15.157:6823/1541443444","heartbeat_front_addr":"172.21.15.157:6821/1541443444","state":["exists","up"]},{"osd":7,"uuid":"495f1013-7888-40e2-8e13-e8568849277b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6824","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6825","nonce":1991105348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6826","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6827","nonce":1991105348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6830","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6831","nonce":1991105348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6828","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6829","nonce":1991105348}]},"public_addr":"172.21.15.157:6825/1991105348","cluster_addr":"172.21.15.157:6827/1991105348","heartbeat_back_addr":"172.21.15.157:6831/1991105348","heartbeat_front_addr":"172.21.15.157:6829/1991105348","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:17:43.350412+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:18:14.269874+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:18:42.389028+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:19:11.897596+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:19:34.298711+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:01.188489+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:26.634387+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:51.945628+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:6801/1900679499":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/307040823":"2024-09-22T17:16:34.462248+0000","172.21.15.18:6800/1900679499":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/3338538266":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/1997413017":"2024-09-22T17:15:42.080516+0000","172.21.15.18:6801/2012086580":"2024-09-22T17:15:42.080516+0000","172.21.15.18:6800/2012086580":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/2286457948":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/3115032843":"2024-09-22T17:15:11.179851+0000","172.21.15.18:6800/902384568":"2024-09-22T17:15:11.179851+0000","172.21.15.18:0/2764100546":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/547361119":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/842545281":"2024-09-22T17:15:11.179851+0000","172.21.15.18:0/3015432587":"2024-09-22T17:15:11.179851+0000","172.21.15.18:6801/902384568":"2024-09-22T17:15:11.179851+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T17:21:12.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:12 smithi018 ceph-mon[23738]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-21T17:21:12.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:12 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/4098125952' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T17:21:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:12 smithi157 ceph-mon[25926]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-21T17:21:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:12 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/4098125952' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T17:21:12.558 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T17:21:12.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T17:21:12.777 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:14.306 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:14 smithi018 ceph-mon[23738]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:14.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:14 smithi157 ceph-mon[25926]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:14.719 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:21:14.720 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":59,"fsid":"ec44ef70-783c-11ef-baf6-efdc52797490","created":"2024-09-21T17:14:47.334351+0000","modified":"2024-09-21T17:20:57.906341+0000","last_up_change":"2024-09-21T17:20:53.920499+0000","last_in_change":"2024-09-21T17:20:36.054112+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T17:18:45.020287+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"ac9a72bf-1285-4f67-b0a4-0e39841eae95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6803","nonce":2412849795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6805","nonce":2412849795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6809","nonce":2412849795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2412849795},{"type":"v1","addr":"172.21.15.18:6807","nonce":2412849795}]},"public_addr":"172.21.15.18:6803/2412849795","cluster_addr":"172.21.15.18:6805/2412849795","heartbeat_back_addr":"172.21.15.18:6809/2412849795","heartbeat_front_addr":"172.21.15.18:6807/2412849795","state":["exists","up"]},{"osd":1,"uuid":"bd92a0a6-7111-4ed3-b70b-20c6dd0a45df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6811","nonce":3829486049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6813","nonce":3829486049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6817","nonce":3829486049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":3829486049},{"type":"v1","addr":"172.21.15.18:6815","nonce":3829486049}]},"public_addr":"172.21.15.18:6811/3829486049","cluster_addr":"172.21.15.18:6813/3829486049","heartbeat_back_addr":"172.21.15.18:6817/3829486049","heartbeat_front_addr":"172.21.15.18:6815/3829486049","state":["exists","up"]},{"osd":2,"uuid":"40e5c057-8087-4fbf-9d48-37a163accaad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6819","nonce":2209132202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6821","nonce":2209132202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6825","nonce":2209132202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":2209132202},{"type":"v1","addr":"172.21.15.18:6823","nonce":2209132202}]},"public_addr":"172.21.15.18:6819/2209132202","cluster_addr":"172.21.15.18:6821/2209132202","heartbeat_back_addr":"172.21.15.18:6825/2209132202","heartbeat_front_addr":"172.21.15.18:6823/2209132202","state":["exists","up"]},{"osd":3,"uuid":"8be04aa4-bfcc-4407-bb16-8500a8f5ae66","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6827","nonce":1384025783}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6829","nonce":1384025783}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6833","nonce":1384025783}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":1384025783},{"type":"v1","addr":"172.21.15.18:6831","nonce":1384025783}]},"public_addr":"172.21.15.18:6827/1384025783","cluster_addr":"172.21.15.18:6829/1384025783","heartbeat_back_addr":"172.21.15.18:6833/1384025783","heartbeat_front_addr":"172.21.15.18:6831/1384025783","state":["exists","up"]},{"osd":4,"uuid":"ef5e6e3f-e4ef-461b-b9df-21dde131c207","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6800","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6801","nonce":2874067171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6802","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6803","nonce":2874067171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6806","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6807","nonce":2874067171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6804","nonce":2874067171},{"type":"v1","addr":"172.21.15.157:6805","nonce":2874067171}]},"public_addr":"172.21.15.157:6801/2874067171","cluster_addr":"172.21.15.157:6803/2874067171","heartbeat_back_addr":"172.21.15.157:6807/2874067171","heartbeat_front_addr":"172.21.15.157:6805/2874067171","state":["exists","up"]},{"osd":5,"uuid":"a0c046bd-29fd-46c6-affd-f7f1c21c8079","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6808","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6809","nonce":3847204715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6810","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6811","nonce":3847204715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6814","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6815","nonce":3847204715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6812","nonce":3847204715},{"type":"v1","addr":"172.21.15.157:6813","nonce":3847204715}]},"public_addr":"172.21.15.157:6809/3847204715","cluster_addr":"172.21.15.157:6811/3847204715","heartbeat_back_addr":"172.21.15.157:6815/3847204715","heartbeat_front_addr":"172.21.15.157:6813/3847204715","state":["exists","up"]},{"osd":6,"uuid":"dc8e90df-1b3e-4b65-8d15-d0beab02c1e9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6816","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6817","nonce":1541443444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6818","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6819","nonce":1541443444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6822","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6823","nonce":1541443444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6820","nonce":1541443444},{"type":"v1","addr":"172.21.15.157:6821","nonce":1541443444}]},"public_addr":"172.21.15.157:6817/1541443444","cluster_addr":"172.21.15.157:6819/1541443444","heartbeat_back_addr":"172.21.15.157:6823/1541443444","heartbeat_front_addr":"172.21.15.157:6821/1541443444","state":["exists","up"]},{"osd":7,"uuid":"495f1013-7888-40e2-8e13-e8568849277b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6824","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6825","nonce":1991105348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6826","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6827","nonce":1991105348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6830","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6831","nonce":1991105348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.157:6828","nonce":1991105348},{"type":"v1","addr":"172.21.15.157:6829","nonce":1991105348}]},"public_addr":"172.21.15.157:6825/1991105348","cluster_addr":"172.21.15.157:6827/1991105348","heartbeat_back_addr":"172.21.15.157:6831/1991105348","heartbeat_front_addr":"172.21.15.157:6829/1991105348","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:17:43.350412+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:18:14.269874+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:18:42.389028+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:19:11.897596+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:19:34.298711+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:01.188489+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:26.634387+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T17:20:51.945628+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:6801/1900679499":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/307040823":"2024-09-22T17:16:34.462248+0000","172.21.15.18:6800/1900679499":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/3338538266":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/1997413017":"2024-09-22T17:15:42.080516+0000","172.21.15.18:6801/2012086580":"2024-09-22T17:15:42.080516+0000","172.21.15.18:6800/2012086580":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/2286457948":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/3115032843":"2024-09-22T17:15:11.179851+0000","172.21.15.18:6800/902384568":"2024-09-22T17:15:11.179851+0000","172.21.15.18:0/2764100546":"2024-09-22T17:16:34.462248+0000","172.21.15.18:0/547361119":"2024-09-22T17:15:42.080516+0000","172.21.15.18:0/842545281":"2024-09-22T17:15:11.179851+0000","172.21.15.18:0/3015432587":"2024-09-22T17:15:11.179851+0000","172.21.15.18:6801/902384568":"2024-09-22T17:15:11.179851+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T17:21:15.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:15 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/4069792065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T17:21:15.409 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T17:21:15.410 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T17:21:15.410 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T17:21:15.411 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T17:21:15.411 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T17:21:15.411 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T17:21:15.412 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T17:21:15.412 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T17:21:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:15 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/4069792065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T17:21:15.853 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:15.854 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:15.856 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:15.857 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:15.857 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:15.858 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:15.862 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:15.863 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:16.249 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:16 smithi018 ceph-mon[23738]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:16 smithi157 ceph-mon[25926]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:18.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:18 smithi018 ceph-mon[23738]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:18.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:18 smithi157 ceph-mon[25926]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:20.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:20 smithi018 ceph-mon[23738]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:20 smithi157 ceph-mon[25926]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:22.335 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:22 smithi018 ceph-mon[23738]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:22.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:22 smithi157 ceph-mon[25926]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:24.316 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:24 smithi018 ceph-mon[23738]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:24.470 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:24 smithi157 ceph-mon[25926]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:24.719 INFO:teuthology.orchestra.run.smithi018.stdout:236223201287 2024-09-21T17:21:24.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T17:21:25.579 INFO:teuthology.orchestra.run.smithi018.stdout:201863462925 2024-09-21T17:21:25.579 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T17:21:26.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:26 smithi018 ceph-mon[23738]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:26 smithi157 ceph-mon[25926]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:26.679 INFO:teuthology.orchestra.run.smithi018.stdout:111669149724 2024-09-21T17:21:26.680 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T17:21:27.652 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:28.258 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:28 smithi018 ceph-mon[23738]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:28 smithi157 ceph-mon[25926]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:29.818 INFO:teuthology.orchestra.run.smithi018.stdout:167503724565 2024-09-21T17:21:29.819 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T17:21:30.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:30 smithi018 ceph-mon[23738]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:30.469 INFO:teuthology.orchestra.run.smithi018.stdout:81604378659 2024-09-21T17:21:30.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T17:21:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:30 smithi157 ceph-mon[25926]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:30.703 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:31.786 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:32.190 INFO:teuthology.orchestra.run.smithi018.stdout:60129542186 2024-09-21T17:21:32.191 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T17:21:32.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:32 smithi018 ceph-mon[23738]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:32 smithi157 ceph-mon[25926]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:32.657 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:33.083 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:34.220 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:34 smithi018 ceph-mon[23738]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:34.247 INFO:teuthology.orchestra.run.smithi018.stdout:133143986202 2024-09-21T17:21:34.248 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T17:21:34.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:34 smithi157 ceph-mon[25926]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:35.492 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:21:35.492 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:21:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:21:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:21:36.292 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:36 smithi018 ceph-mon[23738]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:36 smithi157 ceph-mon[25926]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:37.997 INFO:teuthology.orchestra.run.smithi018.stdout:236223201290 2024-09-21T17:21:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:38 smithi157 ceph-mon[25926]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:38 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/950393675' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T17:21:38.559 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:38 smithi018 ceph-mon[23738]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:38.560 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:38 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/950393675' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T17:21:38.587 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201287 got 236223201290 for osd.7 2024-09-21T17:21:38.588 DEBUG:teuthology.parallel:result is None 2024-09-21T17:21:38.830 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:38.995 INFO:teuthology.orchestra.run.smithi018.stdout:201863462928 2024-09-21T17:21:39.332 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:39 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/4047774727' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T17:21:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:39 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/4047774727' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T17:21:39.623 INFO:teuthology.orchestra.run.smithi018.stdout:111669149727 2024-09-21T17:21:40.223 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:40 smithi018 ceph-mon[23738]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:40.223 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:40 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/337626919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T17:21:40.224 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149727 for osd.3 2024-09-21T17:21:40.224 DEBUG:teuthology.parallel:result is None 2024-09-21T17:21:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:40 smithi157 ceph-mon[25926]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:40 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/337626919' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T17:21:40.879 INFO:teuthology.orchestra.run.smithi018.stdout:38654705713 2024-09-21T17:21:40.880 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T17:21:42.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:42 smithi018 ceph-mon[23738]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:42.527 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:42 smithi157 ceph-mon[25926]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:42.712 INFO:teuthology.orchestra.run.smithi018.stdout:81604378663 2024-09-21T17:21:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:43 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3860874032' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T17:21:43.357 INFO:teuthology.orchestra.run.smithi018.stdout:167503724567 2024-09-21T17:21:43.358 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:43 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3860874032' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T17:21:43.983 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724565 got 167503724567 for osd.5 2024-09-21T17:21:43.983 DEBUG:teuthology.parallel:result is None 2024-09-21T17:21:44.392 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:44 smithi018 ceph-mon[23738]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:44.392 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:44 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3910681260' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T17:21:44.419 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378659 got 81604378663 for osd.2 2024-09-21T17:21:44.419 DEBUG:teuthology.parallel:result is None 2024-09-21T17:21:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:44 smithi157 ceph-mon[25926]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:44 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3910681260' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T17:21:44.737 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:44.738 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:44.768 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462925 got 201863462928 for osd.6 2024-09-21T17:21:44.768 DEBUG:teuthology.parallel:result is None 2024-09-21T17:21:46.236 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:46 smithi018 ceph-mon[23738]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:46.249 INFO:teuthology.orchestra.run.smithi018.stdout:133143986205 2024-09-21T17:21:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:46 smithi157 ceph-mon[25926]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:47 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2459450539' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T17:21:47.534 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:47 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2459450539' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T17:21:47.731 INFO:teuthology.orchestra.run.smithi018.stdout:60129542189 2024-09-21T17:21:48.481 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:48 smithi018 ceph-mon[23738]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:48.481 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:48 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2493740723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T17:21:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:48 smithi157 ceph-mon[25926]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:48 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2493740723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T17:21:48.785 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986202 got 133143986205 for osd.4 2024-09-21T17:21:48.785 DEBUG:teuthology.parallel:result is None 2024-09-21T17:21:49.458 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542186 got 60129542189 for osd.1 2024-09-21T17:21:49.458 DEBUG:teuthology.parallel:result is None 2024-09-21T17:21:49.888 INFO:teuthology.orchestra.run.smithi018.stdout:38654705716 2024-09-21T17:21:50.501 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:50 smithi018 ceph-mon[23738]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:50.501 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:50 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3752239085' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T17:21:50.527 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705713 got 38654705716 for osd.0 2024-09-21T17:21:50.527 DEBUG:teuthology.parallel:result is None 2024-09-21T17:21:50.527 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T17:21:50.527 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T17:21:50.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:50 smithi157 ceph-mon[25926]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:50.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:50 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3752239085' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T17:21:50.743 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:52 smithi018 ceph-mon[23738]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:52 smithi157 ceph-mon[25926]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:52.645 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:21:52.646 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2024-09-21T17:21:53.219 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":206,"stamp":"2024-09-21T17:21:52.570719+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48864,"kb_used_data":6304,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912352,"statfs":{"total":767960285184,"available":767910248448,"internally_reserved":0,"allocated":6455296,"data_stored":3546942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002594"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T17:20:57.912213+0000","last_change":"2024-09-21T17:20:57.912213+0000","last_active":"2024-09-21T17:20:57.912213+0000","last_peered":"2024-09-21T17:20:57.912213+0000","last_clean":"2024-09-21T17:20:57.912213+0000","last_became_active":"2024-09-21T17:20:57.911790+0000","last_became_peered":"2024-09-21T17:20:57.911790+0000","last_unstale":"2024-09-21T17:20:57.912213+0000","last_undegraded":"2024-09-21T17:20:57.912213+0000","last_fullsized":"2024-09-21T17:20:57.912213+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T17:18:45.234584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T17:18:45.234584+0000","last_clean_scrub_stamp":"2024-09-21T17:18:45.234584+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-23T00:25:16.007699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46899999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49399999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.373}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48199999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.54700000000000004}]},{"osd":1,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.47799999999999998}]},{"osd":2,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.58099999999999996}]},{"osd":3,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.60199999999999998}]},{"osd":4,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.443},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.495}]},{"osd":5,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.53600000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]}]},{"osd":1,"up_from":14,"seq":60129542191,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6356,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738796,"statfs":{"total":95995035648,"available":95988527104,"internally_reserved":0,"allocated":602112,"data_stored":244703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:24 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.36399999999999999,"15min":0.33500000000000002},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.221},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.36199999999999999,"15min":0.33600000000000002},"min":{"1min":0.27900000000000003,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 21 17:20:52 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.376,"15min":0.36599999999999999},"min":{"1min":0.29499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.38400000000000001,"15min":0.374},"min":{"1min":0.312,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.66400000000000003,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Sep 21 17:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.46999999999999997,"15min":0.46800000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.48599999999999999,"15min":0.48499999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.495}]},{"osd":4,"last update":"Sat Sep 21 17:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.63500000000000001,"15min":0.63100000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.624,"5min":0.61799999999999999,"15min":0.61699999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.78900000000000003,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 21 17:21:03 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.82099999999999995}]},{"osd":6,"last update":"Sat Sep 21 17:21:31 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.85599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90200000000000002}]}]},{"osd":0,"up_from":9,"seq":38654705717,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 17:21:22 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.34699999999999998,"15min":0.32500000000000001},"min":{"1min":0.29699999999999999,"5min":0.182,"15min":0.182},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.36299999999999999,"15min":0.34000000000000002},"min":{"1min":0.29099999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Sep 21 17:20:49 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.39100000000000001,"15min":0.38300000000000001},"min":{"1min":0.32000000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.433},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.373,"15min":0.36099999999999999},"min":{"1min":0.33900000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 21 17:21:14 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46000000000000002,"15min":0.45900000000000002},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.45400000000000001,"15min":0.44800000000000001},"min":{"1min":0.34100000000000003,"5min":0.245,"15min":0.245},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 21 17:21:40 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.442,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.60699999999999998,"15min":0.60099999999999998},"min":{"1min":0.45000000000000001,"5min":0.443,"15min":0.443},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.77200000000000002}]},{"osd":5,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.81999999999999995}]},{"osd":6,"last update":"Sat Sep 21 17:21:29 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.85499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78500000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378665,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6300,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738852,"statfs":{"total":95995035648,"available":95988584448,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.377,"15min":0.36899999999999999},"min":{"1min":0.26400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.308},{"interface":"front","average":{"1min":0.495,"5min":0.42099999999999999,"15min":0.40899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.41999999999999998}]},{"osd":1,"last update":"Sat Sep 21 17:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.39500000000000002,"15min":0.38700000000000001},"min":{"1min":0.30099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.42999999999999999,"15min":0.41799999999999998},"min":{"1min":0.39200000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Sep 21 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.49099999999999999,"15min":0.495},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64500000000000002,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.503,"15min":0.50600000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 21 17:21:40 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.60899999999999999,"15min":0.60499999999999998},"min":{"1min":0.52900000000000003,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.63800000000000001,"15min":0.63700000000000001},"min":{"1min":0.499,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.59299999999999997}]},{"osd":5,"last update":"Sat Sep 21 17:21:06 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 21 17:21:30 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.622},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.71199999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63400000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:20 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.39600000000000002,"15min":0.39200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 21 17:21:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.441,"15min":0.439},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.318},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.44800000000000001,"15min":0.44400000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 21 17:21:20 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46899999999999997,"15min":0.47299999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.61199999999999999,"5min":0.622,"15min":0.622},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.44,"15min":0.442},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.55400000000000005,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.37}]},{"osd":4,"last update":"Sat Sep 21 17:21:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.94499999999999995,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.624,"15min":0.622},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2010000000000001,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 21 17:21:08 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 21 17:21:32 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.50600000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986207,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:41 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.55000000000000004,"15min":0.54500000000000004},"min":{"1min":0.38900000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78400000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.56699999999999995,"15min":0.57199999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.72499999999999998,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.61499999999999999}]},{"osd":1,"last update":"Sat Sep 21 17:21:41 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.435,"5min":0.37,"15min":0.37},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 21 17:21:41 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56899999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58599999999999997,"15min":0.58499999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 21 17:21:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.59799999999999998,"15min":0.59099999999999997},"min":{"1min":0.46999999999999997,"5min":0.441,"15min":0.441},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.76000000000000001}]},{"osd":5,"last update":"Sat Sep 21 17:21:03 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 21 17:21:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.72899999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65400000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.63500000000000001}]},{"osd":1,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.623},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.73299999999999998}]},{"osd":2,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.77500000000000002}]},{"osd":3,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.749}]},{"osd":4,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 21 17:21:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.59099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T17:21:53.220 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T17:21:53.437 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:54.524 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:54 smithi018 ceph-mon[23738]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:54.524 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:54 smithi018 ceph-mon[23738]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:21:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:54 smithi157 ceph-mon[25926]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:54 smithi157 ceph-mon[25926]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:21:55.250 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:21:55.251 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2024-09-21T17:21:55.906 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":207,"stamp":"2024-09-21T17:21:54.571027+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48864,"kb_used_data":6304,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912352,"statfs":{"total":767960285184,"available":767910248448,"internally_reserved":0,"allocated":6455296,"data_stored":3546942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002626"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T17:20:57.912213+0000","last_change":"2024-09-21T17:20:57.912213+0000","last_active":"2024-09-21T17:20:57.912213+0000","last_peered":"2024-09-21T17:20:57.912213+0000","last_clean":"2024-09-21T17:20:57.912213+0000","last_became_active":"2024-09-21T17:20:57.911790+0000","last_became_peered":"2024-09-21T17:20:57.911790+0000","last_unstale":"2024-09-21T17:20:57.912213+0000","last_undegraded":"2024-09-21T17:20:57.912213+0000","last_fullsized":"2024-09-21T17:20:57.912213+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T17:18:45.234584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T17:18:45.234584+0000","last_clean_scrub_stamp":"2024-09-21T17:18:45.234584+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-23T00:25:16.007699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60599999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51300000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462932,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.748},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57699999999999996}]},{"osd":1,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.72199999999999998}]},{"osd":2,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.60899999999999999}]},{"osd":3,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.64300000000000002}]},{"osd":4,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.435},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Sep 21 17:21:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.46400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542191,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6356,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738796,"statfs":{"total":95995035648,"available":95988527104,"internally_reserved":0,"allocated":602112,"data_stored":244703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:24 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.36399999999999999,"15min":0.33500000000000002},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.221},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.36199999999999999,"15min":0.33600000000000002},"min":{"1min":0.27900000000000003,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 21 17:20:52 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.376,"15min":0.36599999999999999},"min":{"1min":0.29499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.38400000000000001,"15min":0.374},"min":{"1min":0.312,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.66400000000000003,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Sep 21 17:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.46999999999999997,"15min":0.46800000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.48599999999999999,"15min":0.48499999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.495}]},{"osd":4,"last update":"Sat Sep 21 17:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.63500000000000001,"15min":0.63100000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.624,"5min":0.61799999999999999,"15min":0.61699999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.78900000000000003,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 21 17:21:03 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.82099999999999995}]},{"osd":6,"last update":"Sat Sep 21 17:21:31 2024","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.85599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.90200000000000002}]}]},{"osd":0,"up_from":9,"seq":38654705717,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 17:21:22 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.34699999999999998,"15min":0.32500000000000001},"min":{"1min":0.29699999999999999,"5min":0.182,"15min":0.182},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.36299999999999999,"15min":0.34000000000000002},"min":{"1min":0.29099999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Sep 21 17:20:49 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.39100000000000001,"15min":0.38300000000000001},"min":{"1min":0.32000000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.433},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.373,"15min":0.36099999999999999},"min":{"1min":0.33900000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 21 17:21:14 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46000000000000002,"15min":0.45900000000000002},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.45400000000000001,"15min":0.44800000000000001},"min":{"1min":0.34100000000000003,"5min":0.245,"15min":0.245},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 21 17:21:40 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.442,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.60699999999999998,"15min":0.60099999999999998},"min":{"1min":0.45000000000000001,"5min":0.443,"15min":0.443},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.77200000000000002}]},{"osd":5,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.81999999999999995}]},{"osd":6,"last update":"Sat Sep 21 17:21:29 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.85499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78500000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378665,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6300,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738852,"statfs":{"total":95995035648,"available":95988584448,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.377,"15min":0.36899999999999999},"min":{"1min":0.26400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.308},{"interface":"front","average":{"1min":0.495,"5min":0.42099999999999999,"15min":0.40899999999999997},"min":{"1min":0.35299999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.41999999999999998}]},{"osd":1,"last update":"Sat Sep 21 17:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.39500000000000002,"15min":0.38700000000000001},"min":{"1min":0.30099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.42999999999999999,"15min":0.41799999999999998},"min":{"1min":0.39200000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Sep 21 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.49099999999999999,"15min":0.495},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64500000000000002,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.503,"15min":0.50600000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 21 17:21:40 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.60899999999999999,"15min":0.60499999999999998},"min":{"1min":0.52900000000000003,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.63800000000000001,"15min":0.63700000000000001},"min":{"1min":0.499,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.59299999999999997}]},{"osd":5,"last update":"Sat Sep 21 17:21:06 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 21 17:21:30 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.622},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.71199999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63400000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:20 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.39600000000000002,"15min":0.39200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 21 17:21:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.441,"15min":0.439},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.318},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.44800000000000001,"15min":0.44400000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 21 17:21:20 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46899999999999997,"15min":0.47299999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.61199999999999999,"5min":0.622,"15min":0.622},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.44,"15min":0.442},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.55400000000000005,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.37}]},{"osd":4,"last update":"Sat Sep 21 17:21:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.94499999999999995,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.624,"15min":0.622},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.2010000000000001,"5min":1.2010000000000001,"15min":1.2010000000000001},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 21 17:21:08 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 21 17:21:32 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.50600000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986207,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:41 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.55000000000000004,"15min":0.54500000000000004},"min":{"1min":0.38900000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78400000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.56699999999999995,"15min":0.57199999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.72499999999999998,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.61499999999999999}]},{"osd":1,"last update":"Sat Sep 21 17:21:41 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.435,"5min":0.37,"15min":0.37},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 21 17:21:41 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56899999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58599999999999997,"15min":0.58499999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 21 17:21:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.59799999999999998,"15min":0.59099999999999997},"min":{"1min":0.46999999999999997,"5min":0.441,"15min":0.441},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.76000000000000001}]},{"osd":5,"last update":"Sat Sep 21 17:21:03 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 21 17:21:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"last":0.72899999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65400000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724570,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.67800000000000005}]},{"osd":1,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.53600000000000003}]},{"osd":2,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.64600000000000002}]},{"osd":3,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.57499999999999996}]},{"osd":4,"last update":"Sat Sep 21 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.501}]},{"osd":6,"last update":"Sat Sep 21 17:21:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.65800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T17:21:55.907 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T17:21:55.907 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T17:21:55.907 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T17:21:55.908 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T17:21:56.125 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/mon.smithi018/config 2024-09-21T17:21:56.398 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:56 smithi018 ceph-mon[23738]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:56.398 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:21:56.398 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:21:56.398 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:21:56.418 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:56 smithi157 ceph-mon[25926]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:56.418 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:21:56.418 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:21:56.418 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:21:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:57 smithi157 ceph-mon[25926]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:21:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:57 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:21:57.528 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:57 smithi018 ceph-mon[23738]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:21:57.528 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:57 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:21:57.989 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:21:57.989 INFO:teuthology.orchestra.run.smithi018.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T17:21:58.470 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:58 smithi018 ceph-mon[23738]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:58.470 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:21:58 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/4254305515' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T17:21:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:58 smithi157 ceph-mon[25926]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:21:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:21:58 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/4254305515' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T17:21:58.635 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T17:21:58.636 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T17:21:58.636 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T17:21:58.645 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:21:58.645 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph orch status' 2024-09-21T17:22:00.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:00 smithi018 ceph-mon[23738]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:00 smithi157 ceph-mon[25926]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:00.659 INFO:teuthology.orchestra.run.smithi018.stdout:Backend: cephadm 2024-09-21T17:22:00.659 INFO:teuthology.orchestra.run.smithi018.stdout:Available: Yes 2024-09-21T17:22:00.659 INFO:teuthology.orchestra.run.smithi018.stdout:Paused: No 2024-09-21T17:22:01.316 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph orch ps' 2024-09-21T17:22:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:02 smithi157 ceph-mon[25926]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:02 smithi157 ceph-mon[25926]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:02.628 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:02 smithi018 ceph-mon[23738]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:02.628 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:02 smithi018 ceph-mon[23738]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:03.561 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:22:03.561 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (4m) 2m ago 5m 15.4M - ba2b418f427c 960ba9f5795d 2024-09-21T17:22:03.561 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:22:03.561 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (5m) 70s ago 5m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:22:03.561 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (4m) 2m ago 5m 47.1M - 8.3.5 dad864ee21e9 4ac39470ac6e 2024-09-21T17:22:03.561 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:9283 running (7m) 2m ago 7m 448M - 17.2.0 e1d6a67b021e 30cc894916bd 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283 running (5m) 70s ago 5m 411M - 17.2.0 e1d6a67b021e 6a13620a48d3 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (7m) 2m ago 7m 41.2M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (5m) 70s ago 5m 41.9M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (5m) 2m ago 5m 16.6M - 1dbe0e931976 a0d34619a90f 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (4m) 70s ago 4m 15.1M - 1dbe0e931976 cf9822488b33 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (4m) 2m ago 4m 33.9M 3217M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (3m) 2m ago 3m 34.4M 3217M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (3m) 2m ago 3m 32.2M 3217M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (2m) 2m ago 2m 24.9M 3217M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (2m) 70s ago 2m 35.1M 3984M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (2m) 70s ago 2m 32.4M 3984M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (101s) 70s ago 101s 31.5M 3984M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (76s) 70s ago 75s 10.4M 3984M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:22:03.562 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (4m) 2m ago 4m 54.0M - 514e6a882f6e 71c79331fab8 2024-09-21T17:22:04.191 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph orch ls' 2024-09-21T17:22:04.502 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:04 smithi018 ceph-mon[23738]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:04.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:04 smithi157 ceph-mon[25926]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:05 smithi157 ceph-mon[25926]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:05.611 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:05 smithi018 ceph-mon[23738]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:06.369 INFO:teuthology.orchestra.run.smithi018.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-21T17:22:06.370 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-21T17:22:06.370 INFO:teuthology.orchestra.run.smithi018.stdout:crash 2/2 2m ago 6m * 2024-09-21T17:22:06.370 INFO:teuthology.orchestra.run.smithi018.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-21T17:22:06.370 INFO:teuthology.orchestra.run.smithi018.stdout:mgr 2/2 2m ago 6m count:2 2024-09-21T17:22:06.370 INFO:teuthology.orchestra.run.smithi018.stdout:mon 2/2 2m ago 5m smithi018:172.21.15.18=smithi018;smithi157:172.21.15.157=smithi157;count:2 2024-09-21T17:22:06.370 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-21T17:22:06.370 INFO:teuthology.orchestra.run.smithi018.stdout:osd 8 2m ago - 2024-09-21T17:22:06.370 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-21T17:22:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:06 smithi157 ceph-mon[25926]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:06.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:06 smithi018 ceph-mon[23738]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:07.090 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph orch host ls' 2024-09-21T17:22:08.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:08 smithi018 ceph-mon[23738]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:08.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:08 smithi018 ceph-mon[23738]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:08 smithi157 ceph-mon[25926]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:08 smithi157 ceph-mon[25926]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:09.105 INFO:teuthology.orchestra.run.smithi018.stdout:HOST ADDR LABELS STATUS 2024-09-21T17:22:09.105 INFO:teuthology.orchestra.run.smithi018.stdout:smithi018 172.21.15.18 2024-09-21T17:22:09.105 INFO:teuthology.orchestra.run.smithi018.stdout:smithi157 172.21.15.157 2024-09-21T17:22:09.105 INFO:teuthology.orchestra.run.smithi018.stdout:2 hosts in cluster 2024-09-21T17:22:09.756 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph orch device ls' 2024-09-21T17:22:10.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:10 smithi018 ceph-mon[23738]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:10.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:10 smithi018 ceph-mon[23738]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:10.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:10 smithi157 ceph-mon[25926]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:10.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:10 smithi157 ceph-mon[25926]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:11.804 INFO:teuthology.orchestra.run.smithi018.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-21T17:22:11.804 INFO:teuthology.orchestra.run.smithi018.stdout:smithi018 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB75120091480DGN 480G LVM detected, locked 2024-09-21T17:22:11.804 INFO:teuthology.orchestra.run.smithi018.stdout:smithi157 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB7476002V480DGN 480G LVM detected, locked 2024-09-21T17:22:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:12 smithi157 ceph-mon[25926]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:12.528 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:12 smithi018 ceph-mon[23738]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:12.557 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T17:22:12.573 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:22:12.573 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-21T17:22:12.604 INFO:teuthology.orchestra.run.smithi018.stderr:+ systemctl stop nfs-server 2024-09-21T17:22:12.613 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi157.front.sepia.ceph.com 2024-09-21T17:22:12.613 DEBUG:teuthology.orchestra.run.smithi157:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-21T17:22:12.644 INFO:teuthology.orchestra.run.smithi157.stderr:+ systemctl stop nfs-server 2024-09-21T17:22:12.652 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T17:22:12.660 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:22:12.660 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph fs volume create foofs' 2024-09-21T17:22:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:13 smithi157 ceph-mon[25926]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:13.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:13 smithi018 ceph-mon[23738]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:14.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:14 smithi157 ceph-mon[25926]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:14.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:14 smithi018 ceph-mon[23738]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:15 smithi157 ceph-mon[25926]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:15 smithi157 ceph-mon[25926]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:15 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-21T17:22:15.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:15 smithi018 ceph-mon[23738]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:15.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:15 smithi018 ceph-mon[23738]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:15.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:15 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-21T17:22:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:16 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-21T17:22:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:16 smithi157 ceph-mon[25926]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T17:22:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:16 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-21T17:22:16.616 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:16 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[23734]: 2024-09-21T17:22:16.250+0000 7f37f04af700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:22:16.616 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:16 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-21T17:22:16.616 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:16 smithi018 ceph-mon[23738]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T17:22:16.616 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:16 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-21T17:22:17.016 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-21T17:22:17.033 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-21T17:22:17.033 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T17:22:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-21T17:22:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T17:22:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-21T17:22:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:22:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: fsmap foofs:0 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: Saving service mds.foofs spec with placement count:2 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi018.wzitfj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi018.wzitfj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: Deploying daemon mds.foofs.smithi018.wzitfj on smithi018 2024-09-21T17:22:17.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:17 smithi157 ceph-mon[25926]: pgmap v221: 65 pgs: 6 creating+peering, 1 active+clean, 58 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:17.553 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-21T17:22:17.553 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: fsmap foofs:0 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: Saving service mds.foofs spec with placement count:2 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:17.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:17.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:17.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi018.wzitfj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T17:22:17.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi018.wzitfj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T17:22:17.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:17.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: Deploying daemon mds.foofs.smithi018.wzitfj on smithi018 2024-09-21T17:22:17.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:17 smithi018 ceph-mon[23738]: pgmap v221: 65 pgs: 6 creating+peering, 1 active+clean, 58 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:18.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:18 smithi157 ceph-mon[25926]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T17:22:18.540 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:18 smithi018 ceph-mon[23738]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T17:22:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:19 smithi157 ceph-mon[25926]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T17:22:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:19 smithi157 ceph-mon[25926]: pgmap v224: 65 pgs: 2 creating+activating, 9 creating+peering, 10 active+clean, 44 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:19.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:19 smithi018 ceph-mon[23738]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T17:22:19.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:19 smithi018 ceph-mon[23738]: pgmap v224: 65 pgs: 2 creating+activating, 9 creating+peering, 10 active+clean, 44 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:20.357 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:22:20.357 INFO:teuthology.orchestra.run.smithi018.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T17:15:31.093066Z", "last_refresh": "2024-09-21T17:19:22.183460Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T17:15:24.971672Z", "last_refresh": "2024-09-21T17:19:22.183529Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T17:15:27.974654Z", "last_refresh": "2024-09-21T17:19:22.183595Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T17:22:16.277024Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T17:22:16.273406Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T17:15:22.920160Z", "last_refresh": "2024-09-21T17:19:22.183383Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi018:172.21.15.18=smithi018", "smithi157:172.21.15.157=smithi157"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T17:16:15.589289Z", "last_refresh": "2024-09-21T17:19:22.183236Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T17:15:29.301277Z", "last_refresh": "2024-09-21T17:19:22.183660Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T17:19:22.183796Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T17:15:26.707189Z", "last_refresh": "2024-09-21T17:19:22.183724Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T17:22:20.844 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:20.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:21.218 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-21T17:22:22.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:22:22.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: pgmap v225: 65 pgs: 2 creating+activating, 5 creating+peering, 52 active+clean, 6 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi157.iijaxo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi157.iijaxo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: Deploying daemon mds.foofs.smithi157.iijaxo on smithi157 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: mds.? [v2:172.21.15.18:6834/1237994936,v1:172.21.15.18:6835/1237994936] up:boot 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: daemon mds.foofs.smithi018.wzitfj assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: Cluster is now healthy 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: fsmap foofs:0 1 up:standby 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"}]: dispatch 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:creating} 2024-09-21T17:22:22.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:21 smithi157 ceph-mon[25926]: daemon mds.foofs.smithi018.wzitfj is now active in filesystem foofs as rank 0 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: pgmap v225: 65 pgs: 2 creating+activating, 5 creating+peering, 52 active+clean, 6 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi157.iijaxo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi157.iijaxo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: Deploying daemon mds.foofs.smithi157.iijaxo on smithi157 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: mds.? [v2:172.21.15.18:6834/1237994936,v1:172.21.15.18:6835/1237994936] up:boot 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: daemon mds.foofs.smithi018.wzitfj assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: Cluster is now healthy 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: fsmap foofs:0 1 up:standby 2024-09-21T17:22:22.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"}]: dispatch 2024-09-21T17:22:22.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:creating} 2024-09-21T17:22:22.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:21 smithi018 ceph-mon[23738]: daemon mds.foofs.smithi018.wzitfj is now active in filesystem foofs as rank 0 2024-09-21T17:22:22.219 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T17:22:23.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:23 smithi018 ceph-mon[23738]: pgmap v226: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 486 B/s wr, 0 op/s 2024-09-21T17:22:23.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:23 smithi018 ceph-mon[23738]: mds.? [v2:172.21.15.18:6834/1237994936,v1:172.21.15.18:6835/1237994936] up:active 2024-09-21T17:22:23.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:23 smithi018 ceph-mon[23738]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 2024-09-21T17:22:23.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:23.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:23.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:23.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:23 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:23.977 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:23 smithi157 ceph-mon[25926]: pgmap v226: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 486 B/s wr, 0 op/s 2024-09-21T17:22:23.977 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:23 smithi157 ceph-mon[25926]: mds.? [v2:172.21.15.18:6834/1237994936,v1:172.21.15.18:6835/1237994936] up:active 2024-09-21T17:22:23.977 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:23 smithi157 ceph-mon[25926]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 2024-09-21T17:22:23.977 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:23.977 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:23.977 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:23.978 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:23 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:24.413 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:22:24.413 INFO:teuthology.orchestra.run.smithi018.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T17:15:31.093066Z", "last_refresh": "2024-09-21T17:19:22.183460Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T17:15:24.971672Z", "last_refresh": "2024-09-21T17:19:22.183529Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T17:15:27.974654Z", "last_refresh": "2024-09-21T17:19:22.183595Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T17:22:16.277024Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T17:22:16.273406Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T17:15:22.920160Z", "last_refresh": "2024-09-21T17:19:22.183383Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi018:172.21.15.18=smithi018", "smithi157:172.21.15.157=smithi157"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T17:16:15.589289Z", "last_refresh": "2024-09-21T17:19:22.183236Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T17:15:29.301277Z", "last_refresh": "2024-09-21T17:19:22.183660Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T17:19:22.183796Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T17:15:26.707189Z", "last_refresh": "2024-09-21T17:19:22.183724Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T17:22:24.855 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:24 smithi157 ceph-mon[25926]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:boot 2024-09-21T17:22:24.855 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:24 smithi157 ceph-mon[25926]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:22:24.855 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:24 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"}]: dispatch 2024-09-21T17:22:24.855 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:24 smithi157 ceph-mon[25926]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:22:24.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:24 smithi018 ceph-mon[23738]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:boot 2024-09-21T17:22:24.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:24 smithi018 ceph-mon[23738]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:22:24.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:24 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"}]: dispatch 2024-09-21T17:22:24.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:24 smithi018 ceph-mon[23738]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:22:25.503 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-21T17:22:25.796 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:25 smithi018 ceph-mon[23738]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:22:25.796 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:25 smithi018 ceph-mon[23738]: pgmap v227: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 767 B/s wr, 2 op/s 2024-09-21T17:22:25.796 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:25 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:26.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:25 smithi157 ceph-mon[25926]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:22:26.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:25 smithi157 ceph-mon[25926]: pgmap v227: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 767 B/s wr, 2 op/s 2024-09-21T17:22:26.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:25 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:26.503 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T17:22:26.791 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:26 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:27.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:26 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:28.149 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:27 smithi018 ceph-mon[23738]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-21T17:22:28.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:27 smithi157 ceph-mon[25926]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-21T17:22:29.580 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:22:29.580 INFO:teuthology.orchestra.run.smithi018.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T17:15:31.093066Z", "last_refresh": "2024-09-21T17:19:22.183460Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T17:15:24.971672Z", "last_refresh": "2024-09-21T17:19:22.183529Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T17:15:27.974654Z", "last_refresh": "2024-09-21T17:19:22.183595Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T17:22:16.277024Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T17:22:16.273406Z", "last_refresh": "2024-09-21T17:22:25.606935Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T17:15:22.920160Z", "last_refresh": "2024-09-21T17:19:22.183383Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi018:172.21.15.18=smithi018", "smithi157:172.21.15.157=smithi157"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T17:16:15.589289Z", "last_refresh": "2024-09-21T17:19:22.183236Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T17:15:29.301277Z", "last_refresh": "2024-09-21T17:19:22.183660Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T17:19:22.183796Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T17:15:26.707189Z", "last_refresh": "2024-09-21T17:19:22.183724Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T17:22:30.063 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:29 smithi018 ceph-mon[23738]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T17:22:30.063 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:29 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:30.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:29 smithi157 ceph-mon[25926]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T17:22:30.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:29 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:30.504 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-21T17:22:31.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:30 smithi157 ceph-mon[25926]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:22:31.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:30 smithi018 ceph-mon[23738]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:22:31.505 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T17:22:32.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:32 smithi018 ceph-mon[23738]: pgmap v230: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T17:22:32.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:32.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:32.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:32.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:32.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:32.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:32 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:32 smithi157 ceph-mon[25926]: pgmap v230: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T17:22:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:32 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:33.635 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:22:33.635 INFO:teuthology.orchestra.run.smithi018.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T17:15:31.093066Z", "last_refresh": "2024-09-21T17:22:31.067804Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T17:15:24.971672Z", "last_refresh": "2024-09-21T17:22:25.606274Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T17:15:27.974654Z", "last_refresh": "2024-09-21T17:22:31.067949Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T17:22:16.277024Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T17:22:16.273406Z", "last_refresh": "2024-09-21T17:22:25.606935Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T17:15:22.920160Z", "last_refresh": "2024-09-21T17:22:25.606423Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi018:172.21.15.18=smithi018", "smithi157:172.21.15.157=smithi157"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T17:16:15.589289Z", "last_refresh": "2024-09-21T17:22:25.606501Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T17:15:29.301277Z", "last_refresh": "2024-09-21T17:22:25.606573Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T17:22:25.606643Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T17:15:26.707189Z", "last_refresh": "2024-09-21T17:22:31.068082Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T17:22:34.348 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:34 smithi018 ceph-mon[23738]: pgmap v231: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T17:22:34.377 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-21T17:22:34.378 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T17:22:34.395 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:22:34.395 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-21T17:22:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:34 smithi157 ceph-mon[25926]: pgmap v231: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T17:22:35.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:35 smithi018 ceph-mon[23738]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:22:35.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-21T17:22:35.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:35.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:22:35.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:22:35.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:35 smithi157 ceph-mon[25926]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:22:35.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-21T17:22:35.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:35.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:22:35.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:22:36.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:36 smithi018 ceph-mon[23738]: pgmap v232: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 3 op/s 2024-09-21T17:22:36.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:36 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-21T17:22:36.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:36 smithi018 ceph-mon[23738]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T17:22:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:36 smithi157 ceph-mon[25926]: pgmap v232: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 3 op/s 2024-09-21T17:22:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:36 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-21T17:22:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:36 smithi157 ceph-mon[25926]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T17:22:37.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:37 smithi018 ceph-mon[23738]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T17:22:37.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:37 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-21T17:22:37.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:37 smithi157 ceph-mon[25926]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T17:22:37.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:37 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-21T17:22:38.130 INFO:teuthology.orchestra.run.smithi018.stdout:NFS Cluster Created Successfully 2024-09-21T17:22:38.357 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:38 smithi018 ceph-mon[23738]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:38.357 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:38 smithi018 ceph-mon[23738]: pgmap v235: 65 pgs: 1 peering, 64 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:38.357 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:38 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-21T17:22:38.357 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:38 smithi018 ceph-mon[23738]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T17:22:38.358 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:38 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-21T17:22:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:38 smithi157 ceph-mon[25926]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:38 smithi157 ceph-mon[25926]: pgmap v235: 65 pgs: 1 peering, 64 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:38 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-21T17:22:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:38 smithi157 ceph-mon[25926]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T17:22:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:38 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-21T17:22:38.899 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: Saving service nfs.foo spec with placement count:2 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:39.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: Creating key for client.nfs.foo.0.0.smithi018.rwskmy 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T17:22:39.154 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:39 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-21T17:22:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: Saving service nfs.foo spec with placement count:2 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: Creating key for client.nfs.foo.0.0.smithi018.rwskmy 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T17:22:39.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:39.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T17:22:39.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T17:22:39.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T17:22:39.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T17:22:39.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:39 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:40.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:40 smithi018 ceph-mon[23738]: Rados config object exists: conf-nfs.foo 2024-09-21T17:22:40.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:40 smithi018 ceph-mon[23738]: Creating key for client.nfs.foo.0.0.smithi018.rwskmy-rgw 2024-09-21T17:22:40.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:40 smithi018 ceph-mon[23738]: Deploying daemon nfs.foo.0.0.smithi018.rwskmy on smithi018 2024-09-21T17:22:40.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:40 smithi018 ceph-mon[23738]: pgmap v238: 97 pgs: 5 creating+peering, 27 unknown, 1 peering, 64 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:40.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:40 smithi018 ceph-mon[23738]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T17:22:40.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:40 smithi018 ceph-mon[23738]: mgrmap e20: smithi018.jndyyn(active, since 6m), standbys: smithi157.oowooq 2024-09-21T17:22:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:40 smithi157 ceph-mon[25926]: Rados config object exists: conf-nfs.foo 2024-09-21T17:22:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:40 smithi157 ceph-mon[25926]: Creating key for client.nfs.foo.0.0.smithi018.rwskmy-rgw 2024-09-21T17:22:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:40 smithi157 ceph-mon[25926]: Deploying daemon nfs.foo.0.0.smithi018.rwskmy on smithi018 2024-09-21T17:22:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:40 smithi157 ceph-mon[25926]: pgmap v238: 97 pgs: 5 creating+peering, 27 unknown, 1 peering, 64 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T17:22:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:40 smithi157 ceph-mon[25926]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T17:22:40.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:40 smithi157 ceph-mon[25926]: mgrmap e20: smithi018.jndyyn(active, since 6m), standbys: smithi157.oowooq 2024-09-21T17:22:42.368 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:42 smithi018 ceph-mon[23738]: pgmap v240: 97 pgs: 5 creating+peering, 14 unknown, 78 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 227 B/s rd, 455 B/s wr, 1 op/s 2024-09-21T17:22:42.471 INFO:teuthology.orchestra.run.smithi018.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-21T17:22:42.471 INFO:teuthology.orchestra.run.smithi018.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-21T17:22:42.471 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: invalid command 2024-09-21T17:22:42.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:42 smithi157 ceph-mon[25926]: pgmap v240: 97 pgs: 5 creating+peering, 14 unknown, 78 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 227 B/s rd, 455 B/s wr, 1 op/s 2024-09-21T17:22:42.831 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:22:42.831 INFO:teuthology.orchestra.run.smithi018.stdout: "bind": "/fake", 2024-09-21T17:22:42.831 INFO:teuthology.orchestra.run.smithi018.stdout: "fs": "foofs", 2024-09-21T17:22:42.831 INFO:teuthology.orchestra.run.smithi018.stdout: "path": "/", 2024-09-21T17:22:42.831 INFO:teuthology.orchestra.run.smithi018.stdout: "cluster": "foo", 2024-09-21T17:22:42.831 INFO:teuthology.orchestra.run.smithi018.stdout: "mode": "RW" 2024-09-21T17:22:42.831 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:22:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-21T17:22:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-21T17:22:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T17:22:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T17:22:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T17:22:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T17:22:43.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:43 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-21T17:22:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-21T17:22:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T17:22:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T17:22:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T17:22:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T17:22:43.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:43 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:43.672 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-21T17:22:44.325 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:44 smithi018 ceph-mon[23738]: pgmap v241: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T17:22:44.325 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:44 smithi018 ceph-mon[23738]: from='client.14534 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:44.325 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:44 smithi018 ceph-mon[23738]: Creating key for client.nfs.foo.1.0.smithi157.pgvtbt 2024-09-21T17:22:44.325 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:44 smithi018 ceph-mon[23738]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T17:22:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:44 smithi157 ceph-mon[25926]: pgmap v241: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T17:22:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:44 smithi157 ceph-mon[25926]: from='client.14534 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:44 smithi157 ceph-mon[25926]: Creating key for client.nfs.foo.1.0.smithi157.pgvtbt 2024-09-21T17:22:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:44 smithi157 ceph-mon[25926]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T17:22:46.365 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:46 smithi157 ceph-mon[25926]: pgmap v242: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 136 B/s rd, 410 B/s wr, 0 op/s 2024-09-21T17:22:46.365 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T17:22:46.366 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T17:22:46.366 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T17:22:46.366 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T17:22:46.366 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:46 smithi018 ceph-mon[23738]: pgmap v242: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 136 B/s rd, 410 B/s wr, 0 op/s 2024-09-21T17:22:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T17:22:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T17:22:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T17:22:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T17:22:46.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:47.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:47 smithi018 ceph-mon[23738]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:47.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:47 smithi018 ceph-mon[23738]: Rados config object exists: conf-nfs.foo 2024-09-21T17:22:47.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:47 smithi018 ceph-mon[23738]: Creating key for client.nfs.foo.1.0.smithi157.pgvtbt-rgw 2024-09-21T17:22:47.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:47 smithi018 ceph-mon[23738]: Deploying daemon nfs.foo.1.0.smithi157.pgvtbt on smithi157 2024-09-21T17:22:47.441 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:47 smithi157 ceph-mon[25926]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:47.441 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:47 smithi157 ceph-mon[25926]: Rados config object exists: conf-nfs.foo 2024-09-21T17:22:47.441 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:47 smithi157 ceph-mon[25926]: Creating key for client.nfs.foo.1.0.smithi157.pgvtbt-rgw 2024-09-21T17:22:47.441 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:47 smithi157 ceph-mon[25926]: Deploying daemon nfs.foo.1.0.smithi157.pgvtbt on smithi157 2024-09-21T17:22:48.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:48 smithi018 ceph-mon[23738]: pgmap v243: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-21T17:22:48.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:48 smithi018 ceph-mon[23738]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:48 smithi157 ceph-mon[25926]: pgmap v243: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-21T17:22:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:48 smithi157 ceph-mon[25926]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:49 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:49 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:49 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:49 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:49 smithi157 ceph-mon[25926]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:49 smithi157 ceph-mon[25926]: pgmap v244: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-21T17:22:49.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:49 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:49.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:49 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:49.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:49 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:49.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:49 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:49.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:49 smithi018 ceph-mon[23738]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:49.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:49 smithi018 ceph-mon[23738]: pgmap v244: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-21T17:22:50.849 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:50 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:50.850 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:50 smithi157 ceph-mon[25926]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:50.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:50 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:50.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:50 smithi018 ceph-mon[23738]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:52.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:52 smithi018 ceph-mon[23738]: pgmap v245: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.3 KiB/s wr, 6 op/s 2024-09-21T17:22:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:52 smithi018 ceph-mon[23738]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:52.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:52 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:52 smithi157 ceph-mon[25926]: pgmap v245: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.3 KiB/s wr, 6 op/s 2024-09-21T17:22:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:52 smithi157 ceph-mon[25926]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:52 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:53.700 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: pgmap v246: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:54 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:54.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: pgmap v246: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-21T17:22:54.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:22:54.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:22:54.460 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:22:54.460 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.460 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:54 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:22:54.528 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T17:22:54.536 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:22:54.536 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-21T17:22:54.565 INFO:teuthology.orchestra.run.smithi018.stderr:+ mkdir /mnt/foo 2024-09-21T17:22:54.572 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-21T17:22:54.639 INFO:teuthology.orchestra.run.smithi018.stderr:++ hostname 2024-09-21T17:22:54.640 INFO:teuthology.orchestra.run.smithi018.stderr:+ mount -t nfs smithi018:/fake /mnt/foo -o sync 2024-09-21T17:22:55.140 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-21T17:22:55.206 INFO:teuthology.orchestra.run.smithi018.stderr:+ echo test 2024-09-21T17:22:55.216 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-21T17:22:55.280 INFO:teuthology.orchestra.run.smithi018.stderr:+ sync 2024-09-21T17:22:55.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:55 smithi018 ceph-mon[23738]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:55.507 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T17:22:55.517 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T17:22:55.517 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T17:22:55.518 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T17:22:55.518 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:22:55.518 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-21T17:22:55.519 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T17:22:55.519 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-21T17:22:55.520 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T17:22:55.520 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:22:55.520 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-21T17:22:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:55 smithi157 ceph-mon[25926]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:22:55.584 INFO:teuthology.orchestra.run.smithi018.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-21T17:22:55.584 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:22:55.584 INFO:teuthology.orchestra.run.smithi018.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-21T17:22:55.585 INFO:teuthology.orchestra.run.smithi018.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-21T17:22:55.685 INFO:teuthology.orchestra.run.smithi018.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-21T17:22:55.685 INFO:teuthology.orchestra.run.smithi018.stdout:releasing clients 2024-09-21T17:22:56.282 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:56 smithi018 ceph-mon[23738]: pgmap v247: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-21T17:22:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:56 smithi157 ceph-mon[25926]: pgmap v247: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-21T17:22:56.690 INFO:teuthology.orchestra.run.smithi018.stdout: 5 298 54.71 MB/sec warmup 1 sec latency 29.364 ms 2024-09-21T17:22:57.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 589 50.58 MB/sec warmup 2 sec latency 13.881 ms 2024-09-21T17:22:58.326 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:22:58 smithi018 ceph-mon[23738]: pgmap v248: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-09-21T17:22:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:22:58 smithi157 ceph-mon[25926]: pgmap v248: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-09-21T17:22:58.690 INFO:teuthology.orchestra.run.smithi018.stdout: 5 1253 38.42 MB/sec warmup 3 sec latency 15.564 ms 2024-09-21T17:22:59.690 INFO:teuthology.orchestra.run.smithi018.stdout: 5 2394 34.94 MB/sec warmup 4 sec latency 23.607 ms 2024-09-21T17:23:00.041 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-21T17:23:00.338 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:00 smithi018 ceph-mon[23738]: pgmap v249: 97 pgs: 97 active+clean; 14 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-21T17:23:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:00 smithi157 ceph-mon[25926]: pgmap v249: 97 pgs: 97 active+clean; 14 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-21T17:23:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: 5 3346 35.37 MB/sec warmup 5 sec latency 11.464 ms 2024-09-21T17:23:01.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 3950 33.36 MB/sec warmup 6 sec latency 45.007 ms 2024-09-21T17:23:02.374 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:02 smithi018 ceph-mon[23738]: pgmap v250: 97 pgs: 97 active+clean; 86 MiB data, 171 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 7.8 MiB/s wr, 419 op/s 2024-09-21T17:23:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:02 smithi157 ceph-mon[25926]: pgmap v250: 97 pgs: 97 active+clean; 86 MiB data, 171 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 7.8 MiB/s wr, 419 op/s 2024-09-21T17:23:02.690 INFO:teuthology.orchestra.run.smithi018.stdout: 5 4779 30.11 MB/sec warmup 7 sec latency 12.758 ms 2024-09-21T17:23:03.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 5847 29.02 MB/sec warmup 8 sec latency 19.264 ms 2024-09-21T17:23:04.197 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:04 smithi018 ceph-mon[23738]: pgmap v251: 97 pgs: 97 active+clean; 119 MiB data, 209 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 533 op/s 2024-09-21T17:23:04.277 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:04 smithi157 ceph-mon[25926]: pgmap v251: 97 pgs: 97 active+clean; 119 MiB data, 209 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 533 op/s 2024-09-21T17:23:04.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 6556 29.29 MB/sec warmup 9 sec latency 21.277 ms 2024-09-21T17:23:05.568 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T17:23:05.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 7417 29.46 MB/sec warmup 10 sec latency 24.536 ms 2024-09-21T17:23:06.399 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:06 smithi018 ceph-mon[23738]: pgmap v252: 97 pgs: 97 active+clean; 131 MiB data, 241 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 13 MiB/s wr, 778 op/s 2024-09-21T17:23:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:06 smithi157 ceph-mon[25926]: pgmap v252: 97 pgs: 97 active+clean; 131 MiB data, 241 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 13 MiB/s wr, 778 op/s 2024-09-21T17:23:06.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 8171 27.67 MB/sec warmup 11 sec latency 13.407 ms 2024-09-21T17:23:07.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 9172 27.00 MB/sec warmup 12 sec latency 12.577 ms 2024-09-21T17:23:08.168 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:08 smithi018 ceph-mon[23738]: pgmap v253: 97 pgs: 97 active+clean; 137 MiB data, 269 MiB used, 715 GiB / 715 GiB avail; 6.4 KiB/s rd, 15 MiB/s wr, 963 op/s 2024-09-21T17:23:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:08 smithi157 ceph-mon[25926]: pgmap v253: 97 pgs: 97 active+clean; 137 MiB data, 269 MiB used, 715 GiB / 715 GiB avail; 6.4 KiB/s rd, 15 MiB/s wr, 963 op/s 2024-09-21T17:23:08.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 9497 25.46 MB/sec warmup 13 sec latency 54.159 ms 2024-09-21T17:23:09.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 9710 24.20 MB/sec warmup 14 sec latency 39.102 ms 2024-09-21T17:23:10.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:10 smithi018 ceph-mon[23738]: pgmap v254: 97 pgs: 97 active+clean; 137 MiB data, 285 MiB used, 715 GiB / 715 GiB avail; 6.4 KiB/s rd, 16 MiB/s wr, 1.01k op/s 2024-09-21T17:23:10.479 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph mgr module enable nfs --force' 2024-09-21T17:23:10.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:10 smithi157 ceph-mon[25926]: pgmap v254: 97 pgs: 97 active+clean; 137 MiB data, 285 MiB used, 715 GiB / 715 GiB avail; 6.4 KiB/s rd, 16 MiB/s wr, 1.01k op/s 2024-09-21T17:23:10.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 10061 23.84 MB/sec warmup 15 sec latency 108.954 ms 2024-09-21T17:23:11.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 10835 24.20 MB/sec warmup 16 sec latency 41.754 ms 2024-09-21T17:23:12.180 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:12 smithi018 ceph-mon[23738]: pgmap v255: 97 pgs: 97 active+clean; 143 MiB data, 353 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 18 MiB/s wr, 1.28k op/s 2024-09-21T17:23:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:12 smithi157 ceph-mon[25926]: pgmap v255: 97 pgs: 97 active+clean; 143 MiB data, 353 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 18 MiB/s wr, 1.28k op/s 2024-09-21T17:23:12.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 11414 23.14 MB/sec warmup 17 sec latency 13.546 ms 2024-09-21T17:23:13.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 12250 23.19 MB/sec warmup 18 sec latency 16.202 ms 2024-09-21T17:23:14.195 INFO:teuthology.orchestra.run.smithi018.stderr:module 'nfs' is already enabled 2024-09-21T17:23:14.441 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:14 smithi018 ceph-mon[23738]: pgmap v256: 97 pgs: 97 active+clean; 142 MiB data, 406 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 927 op/s 2024-09-21T17:23:14.441 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:14 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2014165922' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T17:23:14.441 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:14 smithi018 ceph-mon[23738]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T17:23:14.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:14 smithi157 ceph-mon[25926]: pgmap v256: 97 pgs: 97 active+clean; 142 MiB data, 406 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 927 op/s 2024-09-21T17:23:14.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:14 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2014165922' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T17:23:14.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:14 smithi157 ceph-mon[25926]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T17:23:14.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 13136 22.59 MB/sec warmup 19 sec latency 18.397 ms 2024-09-21T17:23:15.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:15 smithi018 ceph-mon[23738]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-21T17:23:15.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:15 smithi018 ceph-mon[23738]: mgrmap e21: smithi018.jndyyn(active, since 6m), standbys: smithi157.oowooq 2024-09-21T17:23:15.462 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T17:23:15.527 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:15 smithi157 ceph-mon[25926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-21T17:23:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:15 smithi157 ceph-mon[25926]: mgrmap e21: smithi018.jndyyn(active, since 6m), standbys: smithi157.oowooq 2024-09-21T17:23:15.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 14140 23.65 MB/sec warmup 20 sec latency 23.847 ms 2024-09-21T17:23:16.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:16 smithi018 ceph-mon[23738]: pgmap v257: 97 pgs: 97 active+clean; 152 MiB data, 445 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-09-21T17:23:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:16 smithi157 ceph-mon[25926]: pgmap v257: 97 pgs: 97 active+clean; 152 MiB data, 445 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-09-21T17:23:16.691 INFO:teuthology.orchestra.run.smithi018.stdout: 5 14687 23.20 MB/sec warmup 21 sec latency 18.857 ms 2024-09-21T17:23:17.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 15426 22.75 MB/sec warmup 22 sec latency 12.866 ms 2024-09-21T17:23:18.225 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:18 smithi018 ceph-mon[23738]: pgmap v258: 97 pgs: 97 active+clean; 156 MiB data, 454 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 893 op/s 2024-09-21T17:23:18.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:18 smithi157 ceph-mon[25926]: pgmap v258: 97 pgs: 97 active+clean; 156 MiB data, 454 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 893 op/s 2024-09-21T17:23:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 16294 22.43 MB/sec warmup 23 sec latency 59.290 ms 2024-09-21T17:23:19.057 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe 2024-09-21T17:23:19.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 16765 22.04 MB/sec warmup 24 sec latency 71.837 ms 2024-09-21T17:23:19.957 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T17:23:19.957 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:23:19.957 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-21T17:23:20.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: pgmap v259: 97 pgs: 97 active+clean; 157 MiB data, 475 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 741 op/s 2024-09-21T17:23:20.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:20.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe 2024-09-21T17:23:20.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:20.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:23:20.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:23:20.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:23:20.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:20.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:20 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: pgmap v259: 97 pgs: 97 active+clean; 157 MiB data, 475 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 741 op/s 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:20 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:20.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 17150 21.91 MB/sec warmup 25 sec latency 46.520 ms 2024-09-21T17:23:21.100 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:21 smithi018 ceph-mon[23738]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe 2024-09-21T17:23:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:21 smithi157 ceph-mon[25926]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe 2024-09-21T17:23:21.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 17568 21.61 MB/sec warmup 26 sec latency 71.954 ms 2024-09-21T17:23:22.119 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:22 smithi018 ceph-mon[23738]: pgmap v260: 97 pgs: 97 active+clean; 164 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 978 op/s 2024-09-21T17:23:22.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:22 smithi157 ceph-mon[25926]: pgmap v260: 97 pgs: 97 active+clean; 164 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 978 op/s 2024-09-21T17:23:22.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 17911 21.39 MB/sec warmup 27 sec latency 56.261 ms 2024-09-21T17:23:23.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 18075 20.77 MB/sec warmup 28 sec latency 60.934 ms 2024-09-21T17:23:24.148 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:24 smithi018 ceph-mon[23738]: pgmap v261: 97 pgs: 97 active+clean; 166 MiB data, 589 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 811 op/s 2024-09-21T17:23:24.202 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:23:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:24 smithi157 ceph-mon[25926]: pgmap v261: 97 pgs: 97 active+clean; 166 MiB data, 589 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 811 op/s 2024-09-21T17:23:24.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 18396 20.12 MB/sec warmup 29 sec latency 56.104 ms 2024-09-21T17:23:25.155 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:25 smithi018 ceph-mon[23738]: from='client.24387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:25.321 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:23:25.321 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (5m) 32s ago 7m 22.4M - ba2b418f427c 960ba9f5795d 2024-09-21T17:23:25.321 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (7m) 32s ago 7m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:23:25.321 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (6m) 34s ago 6m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:23:25.321 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (5m) 32s ago 7m 52.0M - 8.3.5 dad864ee21e9 4ac39470ac6e 2024-09-21T17:23:25.321 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (65s) 32s ago 64s 14.8M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (62s) 34s ago 61s 13.4M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:9283 running (8m) 32s ago 8m 460M - 17.2.0 e1d6a67b021e 30cc894916bd 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283 running (6m) 34s ago 6m 412M - 17.2.0 e1d6a67b021e 6a13620a48d3 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (8m) 32s ago 8m 46.4M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (6m) 34s ago 6m 39.5M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (43s) 32s ago 42s 47.0M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (37s) 34s ago 37s 50.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (6m) 32s ago 6m 16.7M - 1dbe0e931976 a0d34619a90f 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (6m) 34s ago 6m 15.5M - 1dbe0e931976 cf9822488b33 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (5m) 32s ago 5m 43.2M 3217M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (5m) 32s ago 5m 42.5M 3217M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (4m) 32s ago 4m 40.3M 3217M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:23:25.322 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (4m) 32s ago 4m 40.0M 3217M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:23:25.323 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (3m) 34s ago 3m 40.7M 3984M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:23:25.323 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (3m) 34s ago 3m 38.3M 3984M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:23:25.323 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (3m) 34s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:23:25.323 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (2m) 34s ago 2m 40.7M 3984M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:23:25.323 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (6m) 32s ago 6m 62.8M - 514e6a882f6e 71c79331fab8 2024-09-21T17:23:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:25 smithi157 ceph-mon[25926]: from='client.24387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:25.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 18676 19.56 MB/sec warmup 30 sec latency 83.713 ms 2024-09-21T17:23:25.912 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:23:25.912 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:23:25.912 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:23:25.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:23:25.912 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:23:25.912 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:23:25.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:23:25.913 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:23:26.163 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:26 smithi018 ceph-mon[23738]: pgmap v262: 97 pgs: 97 active+clean; 173 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 906 op/s 2024-09-21T17:23:26.163 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:26 smithi018 ceph-mon[23738]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:26.163 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:26 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2204691386' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:23:26.353 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:23:26.353 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe", 2024-09-21T17:23:26.353 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:23:26.353 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T17:23:26.353 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "", 2024-09-21T17:23:26.353 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:0ecbb435437226bc091a0c2d4549ba25bf724efe image" 2024-09-21T17:23:26.354 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:23:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:26 smithi157 ceph-mon[25926]: pgmap v262: 97 pgs: 97 active+clean; 173 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 906 op/s 2024-09-21T17:23:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:26 smithi157 ceph-mon[25926]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:26 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2204691386' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:23:26.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 18896 19.12 MB/sec warmup 31 sec latency 117.901 ms 2024-09-21T17:23:26.974 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:23:27.186 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:27 smithi018 ceph-mon[23738]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:27.186 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:27 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1622157742' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:23:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:27 smithi157 ceph-mon[25926]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:27 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/1622157742' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:23:27.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 19345 19.02 MB/sec warmup 32 sec latency 39.817 ms 2024-09-21T17:23:28.197 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:28 smithi018 ceph-mon[23738]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:28.197 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:28 smithi018 ceph-mon[23738]: pgmap v263: 97 pgs: 97 active+clean; 175 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 8.5 MiB/s wr, 723 op/s 2024-09-21T17:23:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:28 smithi157 ceph-mon[25926]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:28 smithi157 ceph-mon[25926]: pgmap v263: 97 pgs: 97 active+clean; 175 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 8.5 MiB/s wr, 723 op/s 2024-09-21T17:23:28.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 19916 18.56 MB/sec warmup 33 sec latency 53.293 ms 2024-09-21T17:23:29.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 20224 18.23 MB/sec warmup 34 sec latency 41.073 ms 2024-09-21T17:23:30.245 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:30 smithi018 ceph-mon[23738]: pgmap v264: 97 pgs: 97 active+clean; 179 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.4 MiB/s wr, 679 op/s 2024-09-21T17:23:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:30 smithi157 ceph-mon[25926]: pgmap v264: 97 pgs: 97 active+clean; 179 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.4 MiB/s wr, 679 op/s 2024-09-21T17:23:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 20968 18.58 MB/sec warmup 35 sec latency 32.685 ms 2024-09-21T17:23:31.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 21662 18.82 MB/sec warmup 36 sec latency 21.356 ms 2024-09-21T17:23:32.367 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:32 smithi018 ceph-mon[23738]: pgmap v265: 97 pgs: 97 active+clean; 174 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.7 MiB/s wr, 784 op/s 2024-09-21T17:23:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:32 smithi157 ceph-mon[25926]: pgmap v265: 97 pgs: 97 active+clean; 174 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.7 MiB/s wr, 784 op/s 2024-09-21T17:23:32.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 22212 18.46 MB/sec warmup 37 sec latency 29.943 ms 2024-09-21T17:23:33.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 22997 18.55 MB/sec warmup 38 sec latency 15.446 ms 2024-09-21T17:23:34.379 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:34 smithi018 ceph-mon[23738]: pgmap v266: 97 pgs: 97 active+clean; 176 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.9 MiB/s wr, 549 op/s 2024-09-21T17:23:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:34 smithi157 ceph-mon[25926]: pgmap v266: 97 pgs: 97 active+clean; 176 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.9 MiB/s wr, 549 op/s 2024-09-21T17:23:34.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 23760 18.34 MB/sec warmup 39 sec latency 27.683 ms 2024-09-21T17:23:35.386 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:23:35.386 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:23:35.386 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-21T17:23:35.386 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-21T17:23:35.386 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:35 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-21T17:23:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:23:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:23:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-21T17:23:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-21T17:23:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:35 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-21T17:23:35.692 INFO:teuthology.orchestra.run.smithi018.stdout: 5 24495 18.70 MB/sec warmup 40 sec latency 21.908 ms 2024-09-21T17:23:36.391 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:36 smithi018 ceph-mon[23738]: pgmap v267: 97 pgs: 97 active+clean; 189 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.5 MiB/s wr, 637 op/s 2024-09-21T17:23:36.391 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:36 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-09-21T17:23:36.391 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:36 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-21T17:23:36.391 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:36 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-21T17:23:36.391 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:36 smithi018 ceph-mon[23738]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T17:23:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:36 smithi157 ceph-mon[25926]: pgmap v267: 97 pgs: 97 active+clean; 189 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.5 MiB/s wr, 637 op/s 2024-09-21T17:23:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:36 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-09-21T17:23:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:36 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-21T17:23:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:36 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-21T17:23:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:36 smithi157 ceph-mon[25926]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T17:23:36.693 INFO:teuthology.orchestra.run.smithi018.stdout: 5 25161 18.86 MB/sec warmup 41 sec latency 23.676 ms 2024-09-21T17:23:37.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:37 smithi018 ceph-mon[23738]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T17:23:37.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:37 smithi157 ceph-mon[25926]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T17:23:37.693 INFO:teuthology.orchestra.run.smithi018.stdout: 5 25778 18.55 MB/sec warmup 42 sec latency 20.733 ms 2024-09-21T17:23:38.162 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:38 smithi018 ceph-mon[23738]: pgmap v270: 97 pgs: 97 active+clean; 194 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 782 op/s 2024-09-21T17:23:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:38 smithi157 ceph-mon[25926]: pgmap v270: 97 pgs: 97 active+clean; 194 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 782 op/s 2024-09-21T17:23:38.693 INFO:teuthology.orchestra.run.smithi018.stdout: 5 26051 18.29 MB/sec warmup 43 sec latency 58.293 ms 2024-09-21T17:23:39.693 INFO:teuthology.orchestra.run.smithi018.stdout: 5 26649 18.21 MB/sec warmup 44 sec latency 32.460 ms 2024-09-21T17:23:40.186 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:40 smithi018 ceph-mon[23738]: pgmap v271: 97 pgs: 97 active+clean; 196 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.9 MiB/s wr, 641 op/s 2024-09-21T17:23:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:40 smithi157 ceph-mon[25926]: pgmap v271: 97 pgs: 97 active+clean; 196 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.9 MiB/s wr, 641 op/s 2024-09-21T17:23:40.693 INFO:teuthology.orchestra.run.smithi018.stdout: 5 27151 17.92 MB/sec warmup 45 sec latency 36.889 ms 2024-09-21T17:23:41.693 INFO:teuthology.orchestra.run.smithi018.stdout: 5 27565 17.86 MB/sec warmup 46 sec latency 44.075 ms 2024-09-21T17:23:42.205 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:42 smithi018 ceph-mon[23738]: pgmap v272: 97 pgs: 97 active+clean; 199 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.7 MiB/s wr, 876 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:42.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:42 smithi157 ceph-mon[25926]: pgmap v272: 97 pgs: 97 active+clean; 199 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.7 MiB/s wr, 876 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:42.699 INFO:teuthology.orchestra.run.smithi018.stdout: 5 28319 18.14 MB/sec warmup 47 sec latency 68.803 ms 2024-09-21T17:23:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: 5 28878 18.12 MB/sec warmup 48 sec latency 43.694 ms 2024-09-21T17:23:44.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:44 smithi018 ceph-mon[23738]: pgmap v273: 97 pgs: 97 active+clean; 202 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.1 MiB/s wr, 861 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:44 smithi157 ceph-mon[25926]: pgmap v273: 97 pgs: 97 active+clean; 202 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.1 MiB/s wr, 861 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: 5 29498 17.92 MB/sec warmup 49 sec latency 65.647 ms 2024-09-21T17:23:45.700 INFO:teuthology.orchestra.run.smithi018.stdout: 5 30173 17.93 MB/sec warmup 50 sec latency 25.302 ms 2024-09-21T17:23:46.248 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:46 smithi018 ceph-mon[23738]: pgmap v274: 97 pgs: 97 active+clean; 206 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.6 MiB/s wr, 886 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:46 smithi157 ceph-mon[25926]: pgmap v274: 97 pgs: 97 active+clean; 206 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.6 MiB/s wr, 886 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:46.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 30830 17.75 MB/sec warmup 51 sec latency 49.415 ms 2024-09-21T17:23:47.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 31559 18.02 MB/sec warmup 52 sec latency 33.066 ms 2024-09-21T17:23:48.261 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:48 smithi018 ceph-mon[23738]: pgmap v275: 97 pgs: 97 active+clean; 211 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.0 MiB/s wr, 745 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:48 smithi157 ceph-mon[25926]: pgmap v275: 97 pgs: 97 active+clean; 211 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.0 MiB/s wr, 745 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:48.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 32168 18.15 MB/sec warmup 53 sec latency 28.799 ms 2024-09-21T17:23:49.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 32597 17.89 MB/sec warmup 54 sec latency 18.763 ms 2024-09-21T17:23:50.362 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:50 smithi018 ceph-mon[23738]: pgmap v276: 97 pgs: 97 active+clean; 212 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.8 MiB/s wr, 686 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:50.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:50 smithi157 ceph-mon[25926]: pgmap v276: 97 pgs: 97 active+clean; 212 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.8 MiB/s wr, 686 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:50.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 33244 17.85 MB/sec warmup 55 sec latency 21.808 ms 2024-09-21T17:23:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 34054 17.74 MB/sec warmup 56 sec latency 17.688 ms 2024-09-21T17:23:52.372 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:52 smithi018 ceph-mon[23738]: pgmap v277: 97 pgs: 97 active+clean; 223 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.7 MiB/s wr, 946 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:52 smithi157 ceph-mon[25926]: pgmap v277: 97 pgs: 97 active+clean; 223 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.7 MiB/s wr, 946 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T17:23:52.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 34830 17.92 MB/sec warmup 57 sec latency 22.937 ms 2024-09-21T17:23:53.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 35752 18.25 MB/sec warmup 58 sec latency 13.487 ms 2024-09-21T17:23:54.384 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:54 smithi018 ceph-mon[23738]: pgmap v278: 97 pgs: 97 active+clean; 219 MiB data, 990 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.6 MiB/s wr, 797 op/s 2024-09-21T17:23:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:54 smithi157 ceph-mon[25926]: pgmap v278: 97 pgs: 97 active+clean; 219 MiB data, 990 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.6 MiB/s wr, 797 op/s 2024-09-21T17:23:54.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 36394 18.04 MB/sec warmup 59 sec latency 20.867 ms 2024-09-21T17:23:55.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 37099 18.07 MB/sec warmup 60 sec latency 24.087 ms 2024-09-21T17:23:56.394 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:56 smithi018 ceph-mon[23738]: pgmap v279: 97 pgs: 97 active+clean; 224 MiB data, 1007 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.7 MiB/s wr, 772 op/s 2024-09-21T17:23:56.394 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:56.394 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:23:56.394 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:56 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:56 smithi157 ceph-mon[25926]: pgmap v279: 97 pgs: 97 active+clean; 224 MiB data, 1007 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.7 MiB/s wr, 772 op/s 2024-09-21T17:23:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:23:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:56 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:23:56.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 37929 17.98 MB/sec warmup 61 sec latency 35.391 ms 2024-09-21T17:23:57.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:57 smithi018 ceph-mon[23738]: Upgrade: Target is version 19.3.0-5129-g0ecbb435 (unknown) 2024-09-21T17:23:57.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:57 smithi018 ceph-mon[23738]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5'] 2024-09-21T17:23:57.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:57 smithi018 ceph-mon[23738]: Upgrade: Need to upgrade myself (mgr.smithi018.jndyyn) 2024-09-21T17:23:57.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:57 smithi018 ceph-mon[23738]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 on smithi157 2024-09-21T17:23:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:57 smithi157 ceph-mon[25926]: Upgrade: Target is version 19.3.0-5129-g0ecbb435 (unknown) 2024-09-21T17:23:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:57 smithi157 ceph-mon[25926]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5'] 2024-09-21T17:23:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:57 smithi157 ceph-mon[25926]: Upgrade: Need to upgrade myself (mgr.smithi018.jndyyn) 2024-09-21T17:23:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:57 smithi157 ceph-mon[25926]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 on smithi157 2024-09-21T17:23:57.569 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:23:57.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 38667 18.18 MB/sec warmup 62 sec latency 19.330 ms 2024-09-21T17:23:58.416 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:58 smithi018 ceph-mon[23738]: pgmap v280: 97 pgs: 97 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 778 op/s 2024-09-21T17:23:58.527 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:58 smithi157 ceph-mon[25926]: pgmap v280: 97 pgs: 97 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 778 op/s 2024-09-21T17:23:58.552 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (6m) 65s ago 7m 22.4M - ba2b418f427c 960ba9f5795d 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (7m) 65s ago 7m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (7m) 67s ago 7m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (6m) 65s ago 7m 52.0M - 8.3.5 dad864ee21e9 4ac39470ac6e 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (98s) 65s ago 97s 14.8M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (95s) 67s ago 95s 13.4M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:9283 running (9m) 65s ago 9m 460M - 17.2.0 e1d6a67b021e 30cc894916bd 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283 running (7m) 67s ago 7m 412M - 17.2.0 e1d6a67b021e 6a13620a48d3 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (9m) 65s ago 9m 46.4M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (7m) 67s ago 7m 39.5M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (76s) 65s ago 75s 47.0M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (70s) 67s ago 70s 50.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:23:58.553 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (7m) 65s ago 7m 16.7M - 1dbe0e931976 a0d34619a90f 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (6m) 67s ago 6m 15.5M - 1dbe0e931976 cf9822488b33 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (6m) 65s ago 6m 43.2M 3217M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (5m) 65s ago 5m 42.5M 3217M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (5m) 65s ago 5m 40.3M 3217M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (4m) 65s ago 4m 40.0M 3217M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (4m) 67s ago 4m 40.7M 3984M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (4m) 67s ago 4m 38.3M 3984M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (3m) 67s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:23:58.554 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (3m) 67s ago 3m 40.7M 3984M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:23:58.555 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (6m) 65s ago 6m 62.8M - 514e6a882f6e 71c79331fab8 2024-09-21T17:23:58.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 39175 18.23 MB/sec warmup 63 sec latency 43.218 ms 2024-09-21T17:23:59.171 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:23:59.171 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:23:59.171 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:23:59.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:23:59.171 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:23:59.171 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:23:59.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:23:59.172 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:23:59.423 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:59 smithi018 ceph-mon[23738]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:59.423 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:59 smithi018 ceph-mon[23738]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:59.423 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:23:59 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/23255923' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:23:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:59 smithi157 ceph-mon[25926]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:59 smithi157 ceph-mon[25926]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:23:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:23:59 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/23255923' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:23:59.651 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:23:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:23:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:23:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T17:23:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "0/23 daemons upgraded", 2024-09-21T17:23:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 image on host smithi157" 2024-09-21T17:23:59.652 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:23:59.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 39608 18.08 MB/sec warmup 64 sec latency 41.555 ms 2024-09-21T17:24:00.221 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:24:00.432 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:00 smithi018 ceph-mon[23738]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:00.432 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:00 smithi018 ceph-mon[23738]: pgmap v281: 97 pgs: 97 active+clean; 231 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 762 op/s 2024-09-21T17:24:00.527 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:00 smithi157 ceph-mon[25926]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:00 smithi157 ceph-mon[25926]: pgmap v281: 97 pgs: 97 active+clean; 231 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 762 op/s 2024-09-21T17:24:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: 5 40015 17.87 MB/sec warmup 65 sec latency 109.211 ms 2024-09-21T17:24:01.439 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:01 smithi018 ceph-mon[23738]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:01.439 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:01 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/588847279' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:24:01.527 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:01 smithi157 ceph-mon[25926]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:01 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/588847279' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:24:01.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 40624 17.91 MB/sec warmup 66 sec latency 44.186 ms 2024-09-21T17:24:02.351 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:02 smithi018 ceph-mon[23738]: pgmap v282: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-09-21T17:24:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:02 smithi157 ceph-mon[25926]: pgmap v282: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-09-21T17:24:02.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 41196 17.70 MB/sec warmup 67 sec latency 41.701 ms 2024-09-21T17:24:03.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 41522 17.55 MB/sec warmup 68 sec latency 53.499 ms 2024-09-21T17:24:04.239 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:04 smithi018 ceph-mon[23738]: pgmap v283: 97 pgs: 97 active+clean; 247 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 849 op/s 2024-09-21T17:24:04.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:04 smithi157 ceph-mon[25926]: pgmap v283: 97 pgs: 97 active+clean; 247 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 849 op/s 2024-09-21T17:24:04.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 42021 17.63 MB/sec warmup 69 sec latency 37.019 ms 2024-09-21T17:24:05.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 42355 17.55 MB/sec warmup 70 sec latency 97.786 ms 2024-09-21T17:24:06.289 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:06 smithi018 ceph-mon[23738]: pgmap v284: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 841 op/s 2024-09-21T17:24:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:06 smithi157 ceph-mon[25926]: pgmap v284: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 841 op/s 2024-09-21T17:24:06.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 42706 17.53 MB/sec warmup 71 sec latency 212.212 ms 2024-09-21T17:24:07.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 42926 17.38 MB/sec warmup 72 sec latency 59.243 ms 2024-09-21T17:24:08.346 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:08 smithi018 ceph-mon[23738]: pgmap v285: 97 pgs: 97 active+clean; 243 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 783 op/s 2024-09-21T17:24:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:08 smithi157 ceph-mon[25926]: pgmap v285: 97 pgs: 97 active+clean; 243 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 783 op/s 2024-09-21T17:24:08.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 43097 17.15 MB/sec warmup 73 sec latency 72.155 ms 2024-09-21T17:24:09.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 43299 16.94 MB/sec warmup 74 sec latency 99.323 ms 2024-09-21T17:24:10.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:10 smithi018 ceph-mon[23738]: pgmap v286: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 806 op/s 2024-09-21T17:24:10.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:10 smithi157 ceph-mon[25926]: pgmap v286: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 806 op/s 2024-09-21T17:24:10.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 43470 16.74 MB/sec warmup 75 sec latency 78.539 ms 2024-09-21T17:24:11.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 43607 16.57 MB/sec warmup 76 sec latency 68.031 ms 2024-09-21T17:24:12.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:12 smithi018 ceph-mon[23738]: pgmap v287: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 804 op/s 2024-09-21T17:24:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:12 smithi157 ceph-mon[25926]: pgmap v287: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 804 op/s 2024-09-21T17:24:12.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 43838 16.46 MB/sec warmup 77 sec latency 68.867 ms 2024-09-21T17:24:13.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 44074 16.31 MB/sec warmup 78 sec latency 77.814 ms 2024-09-21T17:24:14.458 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:14 smithi018 ceph-mon[23738]: pgmap v288: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.6 MiB/s wr, 502 op/s 2024-09-21T17:24:14.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:14 smithi157 ceph-mon[25926]: pgmap v288: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.6 MiB/s wr, 502 op/s 2024-09-21T17:24:14.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 44527 16.20 MB/sec warmup 79 sec latency 67.229 ms 2024-09-21T17:24:15.702 INFO:teuthology.orchestra.run.smithi018.stdout: 5 44869 16.05 MB/sec warmup 80 sec latency 83.531 ms 2024-09-21T17:24:16.510 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:16 smithi018 ceph-mon[23738]: pgmap v289: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 510 op/s 2024-09-21T17:24:16.527 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:16 smithi157 ceph-mon[25926]: pgmap v289: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 510 op/s 2024-09-21T17:24:16.703 INFO:teuthology.orchestra.run.smithi018.stdout: 5 45116 15.93 MB/sec warmup 81 sec latency 74.333 ms 2024-09-21T17:24:17.703 INFO:teuthology.orchestra.run.smithi018.stdout: 5 45677 16.05 MB/sec warmup 82 sec latency 39.257 ms 2024-09-21T17:24:18.039 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:17 smithi018 ceph-mon[23738]: pgmap v290: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 539 op/s 2024-09-21T17:24:18.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:17 smithi157 ceph-mon[25926]: pgmap v290: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 539 op/s 2024-09-21T17:24:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: 5 46222 16.13 MB/sec warmup 83 sec latency 60.492 ms 2024-09-21T17:24:19.703 INFO:teuthology.orchestra.run.smithi018.stdout: 5 46476 16.03 MB/sec warmup 84 sec latency 39.139 ms 2024-09-21T17:24:20.119 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:19 smithi018 ceph-mon[23738]: pgmap v291: 97 pgs: 97 active+clean; 254 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 437 op/s 2024-09-21T17:24:20.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:19 smithi157 ceph-mon[25926]: pgmap v291: 97 pgs: 97 active+clean; 254 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 437 op/s 2024-09-21T17:24:20.703 INFO:teuthology.orchestra.run.smithi018.stdout: 5 46881 15.87 MB/sec warmup 85 sec latency 64.479 ms 2024-09-21T17:24:21.703 INFO:teuthology.orchestra.run.smithi018.stdout: 5 47123 15.73 MB/sec warmup 86 sec latency 96.897 ms 2024-09-21T17:24:22.219 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:21 smithi018 ceph-mon[23738]: pgmap v292: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.3 MiB/s wr, 354 op/s 2024-09-21T17:24:22.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:21 smithi157 ceph-mon[25926]: pgmap v292: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.3 MiB/s wr, 354 op/s 2024-09-21T17:24:22.703 INFO:teuthology.orchestra.run.smithi018.stdout: 5 47435 15.65 MB/sec warmup 87 sec latency 53.911 ms 2024-09-21T17:24:23.710 INFO:teuthology.orchestra.run.smithi018.stdout: 5 48001 15.62 MB/sec warmup 88 sec latency 37.655 ms 2024-09-21T17:24:23.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:23 smithi018 ceph-mon[23738]: pgmap v293: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 466 op/s 2024-09-21T17:24:24.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:23 smithi157 ceph-mon[25926]: pgmap v293: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 466 op/s 2024-09-21T17:24:24.710 INFO:teuthology.orchestra.run.smithi018.stdout: 5 48451 15.51 MB/sec warmup 89 sec latency 61.658 ms 2024-09-21T17:24:25.716 INFO:teuthology.orchestra.run.smithi018.stdout: 5 48740 15.44 MB/sec warmup 90 sec latency 66.750 ms 2024-09-21T17:24:26.042 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:25 smithi018 ceph-mon[23738]: pgmap v294: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 517 op/s 2024-09-21T17:24:26.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:25 smithi157 ceph-mon[25926]: pgmap v294: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 517 op/s 2024-09-21T17:24:26.716 INFO:teuthology.orchestra.run.smithi018.stdout: 5 49386 15.56 MB/sec warmup 91 sec latency 94.507 ms 2024-09-21T17:24:27.571 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:27 smithi018 ceph-mon[23738]: Upgrade: Updating mgr.smithi157.oowooq 2024-09-21T17:24:27.571 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:27 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:27.571 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:27 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:24:27.571 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:27 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:24:27.571 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:27 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:24:27.572 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:27 smithi018 ceph-mon[23738]: Deploying daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:24:27.572 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:27 smithi018 ceph-mon[23738]: pgmap v295: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.9 MiB/s wr, 493 op/s 2024-09-21T17:24:27.716 INFO:teuthology.orchestra.run.smithi018.stdout: 5 49835 15.63 MB/sec warmup 92 sec latency 129.473 ms 2024-09-21T17:24:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:27 smithi157 ceph-mon[25926]: Upgrade: Updating mgr.smithi157.oowooq 2024-09-21T17:24:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:27 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:27 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:24:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:27 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:24:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:27 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:24:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:27 smithi157 ceph-mon[25926]: Deploying daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:24:27.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:27 smithi157 ceph-mon[25926]: pgmap v295: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.9 MiB/s wr, 493 op/s 2024-09-21T17:24:28.716 INFO:teuthology.orchestra.run.smithi018.stdout: 5 50023 15.51 MB/sec warmup 93 sec latency 73.070 ms 2024-09-21T17:24:29.716 INFO:teuthology.orchestra.run.smithi018.stdout: 5 50270 15.36 MB/sec warmup 94 sec latency 43.684 ms 2024-09-21T17:24:29.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:29 smithi157 ceph-mon[25926]: pgmap v296: 97 pgs: 97 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 467 op/s 2024-09-21T17:24:29.897 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:29 smithi018 ceph-mon[23738]: pgmap v296: 97 pgs: 97 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 467 op/s 2024-09-21T17:24:30.639 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:24:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: 5 50556 15.23 MB/sec warmup 95 sec latency 120.693 ms 2024-09-21T17:24:31.486 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:24:31.486 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (6m) 98s ago 8m 22.4M - ba2b418f427c 960ba9f5795d 2024-09-21T17:24:31.486 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (8m) 98s ago 8m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:24:31.486 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (7m) 100s ago 7m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (6m) 98s ago 8m 52.0M - 8.3.5 dad864ee21e9 4ac39470ac6e 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (2m) 98s ago 2m 14.8M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (2m) 100s ago 2m 13.4M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:9283 running (9m) 98s ago 9m 460M - 17.2.0 e1d6a67b021e 30cc894916bd 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283 running (7m) 100s ago 7m 412M - 17.2.0 e1d6a67b021e 6a13620a48d3 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (9m) 98s ago 9m 46.4M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (7m) 100s ago 7m 39.5M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (109s) 98s ago 108s 47.0M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (103s) 100s ago 103s 50.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (8m) 98s ago 8m 16.7M - 1dbe0e931976 a0d34619a90f 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (7m) 100s ago 7m 15.5M - 1dbe0e931976 cf9822488b33 2024-09-21T17:24:31.487 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (6m) 98s ago 6m 43.2M 3217M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:24:31.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (6m) 98s ago 6m 42.5M 3217M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:24:31.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (5m) 98s ago 5m 40.3M 3217M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:24:31.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (5m) 98s ago 5m 40.0M 3217M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:24:31.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (5m) 100s ago 5m 40.7M 3984M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:24:31.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (4m) 100s ago 4m 38.3M 3984M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:24:31.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (4m) 100s ago 4m 40.1M 3984M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:24:31.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (3m) 100s ago 3m 40.7M 3984M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:24:31.488 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (7m) 98s ago 7m 62.8M - 514e6a882f6e 71c79331fab8 2024-09-21T17:24:31.718 INFO:teuthology.orchestra.run.smithi018.stdout: 5 50901 15.19 MB/sec warmup 96 sec latency 97.298 ms 2024-09-21T17:24:31.922 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:31 smithi018 ceph-mon[23738]: pgmap v297: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.7 MiB/s wr, 638 op/s 2024-09-21T17:24:31.922 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:31 smithi018 ceph-mon[23738]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:31.922 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:31 smithi018 ceph-mon[23738]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:31 smithi157 ceph-mon[25926]: pgmap v297: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.7 MiB/s wr, 638 op/s 2024-09-21T17:24:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:31 smithi157 ceph-mon[25926]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:31 smithi157 ceph-mon[25926]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:32.050 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:24:32.050 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:24:32.050 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:24:32.050 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:24:32.050 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:24:32.051 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:24:32.498 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:24:32.498 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:24:32.498 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:24:32.498 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T17:24:32.498 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "0/23 daemons upgraded", 2024-09-21T17:24:32.498 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mgr daemons" 2024-09-21T17:24:32.498 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:24:32.726 INFO:teuthology.orchestra.run.smithi018.stdout: 5 51545 15.16 MB/sec warmup 97 sec latency 30.598 ms 2024-09-21T17:24:33.119 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:24:33.178 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:32 smithi018 ceph-mon[23738]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:33.178 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:32 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/2822418503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:24:33.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:32 smithi157 ceph-mon[25926]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:33.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:32 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/2822418503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:24:33.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 52187 15.13 MB/sec warmup 98 sec latency 43.024 ms 2024-09-21T17:24:34.199 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:33 smithi018 ceph-mon[23738]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:34.199 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:33 smithi018 ceph-mon[23738]: pgmap v298: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 570 op/s 2024-09-21T17:24:34.199 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:33 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/760163567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:24:34.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:33 smithi157 ceph-mon[25926]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:24:34.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:33 smithi157 ceph-mon[25926]: pgmap v298: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 570 op/s 2024-09-21T17:24:34.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:33 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/760163567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:24:34.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 52723 15.21 MB/sec warmup 99 sec latency 91.589 ms 2024-09-21T17:24:35.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:24:35.004 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:34 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:24:35.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:24:35.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:34 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:24:35.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 53197 15.23 MB/sec warmup 100 sec latency 40.052 ms 2024-09-21T17:24:36.019 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:35 smithi018 ceph-mon[23738]: pgmap v299: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.6 MiB/s wr, 557 op/s 2024-09-21T17:24:36.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:35 smithi157 ceph-mon[25926]: pgmap v299: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.6 MiB/s wr, 557 op/s 2024-09-21T17:24:36.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 53595 15.22 MB/sec warmup 101 sec latency 37.353 ms 2024-09-21T17:24:37.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 54007 15.10 MB/sec warmup 102 sec latency 40.721 ms 2024-09-21T17:24:38.071 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:38 smithi018 ceph-mon[23738]: pgmap v300: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 584 op/s 2024-09-21T17:24:38.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:38 smithi157 ceph-mon[25926]: pgmap v300: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 584 op/s 2024-09-21T17:24:38.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 54301 15.00 MB/sec warmup 103 sec latency 77.017 ms 2024-09-21T17:24:39.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 54937 15.05 MB/sec warmup 104 sec latency 32.445 ms 2024-09-21T17:24:40.164 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:40 smithi018 ceph-mon[23738]: pgmap v301: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.2 MiB/s wr, 596 op/s 2024-09-21T17:24:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:40 smithi157 ceph-mon[25926]: pgmap v301: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.2 MiB/s wr, 596 op/s 2024-09-21T17:24:40.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 55531 14.96 MB/sec warmup 105 sec latency 30.074 ms 2024-09-21T17:24:41.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 55999 14.98 MB/sec warmup 106 sec latency 37.444 ms 2024-09-21T17:24:42.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:42 smithi018 ceph-mon[23738]: pgmap v302: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 677 op/s 2024-09-21T17:24:42.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:42.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:24:42.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:24:42.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:42 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:24:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:42 smithi157 ceph-mon[25926]: pgmap v302: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 677 op/s 2024-09-21T17:24:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:24:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:24:42.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:42 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:24:42.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 56398 15.05 MB/sec warmup 107 sec latency 97.077 ms 2024-09-21T17:24:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 56762 14.99 MB/sec warmup 108 sec latency 56.266 ms 2024-09-21T17:24:44.060 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:44 smithi157 ceph-mon[25926]: pgmap v303: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.1 MiB/s wr, 529 op/s 2024-09-21T17:24:44.259 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:44 smithi018 ceph-mon[23738]: pgmap v303: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.1 MiB/s wr, 529 op/s 2024-09-21T17:24:44.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 57052 14.97 MB/sec warmup 109 sec latency 81.884 ms 2024-09-21T17:24:45.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 57268 14.85 MB/sec warmup 110 sec latency 61.284 ms 2024-09-21T17:24:46.056 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:46 smithi018 ceph-mon[23738]: pgmap v304: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 726 op/s 2024-09-21T17:24:46.056 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:46.056 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:46 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:46 smithi157 ceph-mon[25926]: pgmap v304: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 726 op/s 2024-09-21T17:24:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:46.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:46 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:46.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 57588 14.75 MB/sec warmup 111 sec latency 83.388 ms 2024-09-21T17:24:47.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 57978 14.71 MB/sec warmup 112 sec latency 51.115 ms 2024-09-21T17:24:48.106 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:48 smithi018 ceph-mon[23738]: pgmap v305: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 622 op/s 2024-09-21T17:24:48.106 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:48 smithi018 ceph-mon[23738]: Standby manager daemon smithi157.oowooq restarted 2024-09-21T17:24:48.106 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:48 smithi018 ceph-mon[23738]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:24:48.106 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:48 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:24:48.106 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:48 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:24:48.106 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:48 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:24:48.107 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:48 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:24:48.167 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:48 smithi157 ceph-mon[25926]: pgmap v305: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 622 op/s 2024-09-21T17:24:48.167 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:48 smithi157 ceph-mon[25926]: Standby manager daemon smithi157.oowooq restarted 2024-09-21T17:24:48.167 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:48 smithi157 ceph-mon[25926]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:24:48.167 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:48 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:24:48.167 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:48 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:24:48.167 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:48 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:24:48.167 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:48 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:24:48.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 58430 14.67 MB/sec warmup 113 sec latency 59.842 ms 2024-09-21T17:24:49.127 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:49 smithi018 ceph-mon[23738]: mgrmap e22: smithi018.jndyyn(active, since 8m), standbys: smithi157.oowooq 2024-09-21T17:24:49.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:49 smithi157 ceph-mon[25926]: mgrmap e22: smithi018.jndyyn(active, since 8m), standbys: smithi157.oowooq 2024-09-21T17:24:49.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 58849 14.60 MB/sec warmup 114 sec latency 52.521 ms 2024-09-21T17:24:50.149 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:50 smithi018 ceph-mon[23738]: pgmap v306: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.0 MiB/s wr, 557 op/s 2024-09-21T17:24:50.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:50 smithi157 ceph-mon[25926]: pgmap v306: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.0 MiB/s wr, 557 op/s 2024-09-21T17:24:50.737 INFO:teuthology.orchestra.run.smithi018.stdout: 5 59273 14.55 MB/sec warmup 115 sec latency 63.310 ms 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: Upgrade: Need to upgrade myself (mgr.smithi018.jndyyn) 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: Failing over to other MGR 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr fail", "who": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T17:24:51.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:51 smithi018 ceph-mon[23738]: Standby manager daemon smithi018.jndyyn started 2024-09-21T17:24:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' 2024-09-21T17:24:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:24:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: Upgrade: Need to upgrade myself (mgr.smithi018.jndyyn) 2024-09-21T17:24:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: Failing over to other MGR 2024-09-21T17:24:51.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr fail", "who": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:24:51.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T17:24:51.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:51 smithi157 ceph-mon[25926]: Standby manager daemon smithi018.jndyyn started 2024-09-21T17:24:51.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 59558 14.52 MB/sec warmup 116 sec latency 61.516 ms 2024-09-21T17:24:52.150 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "mgr fail", "who": "smithi018.jndyyn"}]': finished 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: mgrmap e23: smithi157.oowooq(active, starting, since 1.00046s), standbys: smithi018.jndyyn 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr metadata", "who": "smithi157.oowooq", "id": "smithi157.oowooq"}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:24:52.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: Manager daemon smithi157.oowooq is now available 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: Checking for cert/key for grafana.smithi018 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:24:52.152 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi157.oowooq/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:24:52.153 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi157.oowooq/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:24:52.153 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi157.oowooq/trash_purge_schedule"}]: dispatch 2024-09-21T17:24:52.153 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:52 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi157.oowooq/trash_purge_schedule"}]: dispatch 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.14200 172.21.15.18:0/3665795120' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "mgr fail", "who": "smithi018.jndyyn"}]': finished 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: mgrmap e23: smithi157.oowooq(active, starting, since 1.00046s), standbys: smithi018.jndyyn 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"}]: dispatch 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"}]: dispatch 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr metadata", "who": "smithi157.oowooq", "id": "smithi157.oowooq"}]: dispatch 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:24:52.209 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: Manager daemon smithi157.oowooq is now available 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:24:52.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: Checking for cert/key for grafana.smithi018 2024-09-21T17:24:52.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:24:52.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:24:52.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:24:52.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi157.oowooq/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:24:52.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi157.oowooq/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:24:52.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi157.oowooq/trash_purge_schedule"}]: dispatch 2024-09-21T17:24:52.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:52 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi157.oowooq/trash_purge_schedule"}]: dispatch 2024-09-21T17:24:52.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 59922 14.57 MB/sec warmup 117 sec latency 45.624 ms 2024-09-21T17:24:53.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:53 smithi018 ceph-mon[23738]: mgrmap e24: smithi157.oowooq(active, since 2s), standbys: smithi018.jndyyn 2024-09-21T17:24:53.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:53 smithi018 ceph-mon[23738]: pgmap v3: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T17:24:53.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:53 smithi018 ceph-mon[23738]: Deploying cephadm binary to smithi157 2024-09-21T17:24:53.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:53 smithi018 ceph-mon[23738]: Deploying cephadm binary to smithi018 2024-09-21T17:24:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:53 smithi157 ceph-mon[25926]: mgrmap e24: smithi157.oowooq(active, since 2s), standbys: smithi018.jndyyn 2024-09-21T17:24:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:53 smithi157 ceph-mon[25926]: pgmap v3: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T17:24:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:53 smithi157 ceph-mon[25926]: Deploying cephadm binary to smithi157 2024-09-21T17:24:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:53 smithi157 ceph-mon[25926]: Deploying cephadm binary to smithi018 2024-09-21T17:24:53.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 60203 14.50 MB/sec warmup 118 sec latency 89.026 ms 2024-09-21T17:24:54.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:54 smithi018 ceph-mon[23738]: pgmap v4: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T17:24:54.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:54 smithi018 ceph-mon[23738]: [21/Sep/2024:17:24:53] ENGINE Bus STARTING 2024-09-21T17:24:54.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:54 smithi018 ceph-mon[23738]: [21/Sep/2024:17:24:53] ENGINE Serving on http://172.21.15.157:8765 2024-09-21T17:24:54.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:54 smithi018 ceph-mon[23738]: [21/Sep/2024:17:24:53] ENGINE Serving on https://172.21.15.157:7150 2024-09-21T17:24:54.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:54 smithi018 ceph-mon[23738]: [21/Sep/2024:17:24:53] ENGINE Bus STARTED 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:54 smithi018 ceph-mon[23738]: [21/Sep/2024:17:24:53] ENGINE Error in HTTPServer.serve 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Traceback (most recent call last): 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._run(expiration_interval) 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T17:24:54.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T17:24:54.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self.do_handshake() 2024-09-21T17:24:54.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T17:24:54.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T17:24:54.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:54 smithi157 ceph-mon[25926]: pgmap v4: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:54 smithi157 ceph-mon[25926]: [21/Sep/2024:17:24:53] ENGINE Bus STARTING 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:54 smithi157 ceph-mon[25926]: [21/Sep/2024:17:24:53] ENGINE Serving on http://172.21.15.157:8765 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:54 smithi157 ceph-mon[25926]: [21/Sep/2024:17:24:53] ENGINE Serving on https://172.21.15.157:7150 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:54 smithi157 ceph-mon[25926]: [21/Sep/2024:17:24:53] ENGINE Bus STARTED 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:54 smithi157 ceph-mon[25926]: [21/Sep/2024:17:24:53] ENGINE Error in HTTPServer.serve 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Traceback (most recent call last): 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._connections.run(self.expiration_interval) 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._run(expiration_interval) 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T17:24:54.419 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: s = self.context.wrap_socket( 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: return self.sslsocket_class._create( 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self.do_handshake() 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._sslobj.do_handshake() 2024-09-21T17:24:54.420 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T17:24:54.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 60511 14.47 MB/sec warmup 119 sec latency 255.811 ms 2024-09-21T17:24:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:55 smithi157 ceph-mon[25926]: mgrmap e25: smithi157.oowooq(active, since 4s), standbys: smithi018.jndyyn 2024-09-21T17:24:55.279 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:55 smithi018 ceph-mon[23738]: mgrmap e25: smithi157.oowooq(active, since 4s), standbys: smithi018.jndyyn 2024-09-21T17:24:56.316 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:56 smithi157 ceph-mon[25926]: pgmap v5: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T17:24:56.324 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:56 smithi018 ceph-mon[23738]: pgmap v5: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T17:24:56.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 60994 5.47 MB/sec execute 1 sec latency 115.833 ms 2024-09-21T17:24:57.328 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:57 smithi018 ceph-mon[23738]: mgrmap e26: smithi157.oowooq(active, since 6s), standbys: smithi018.jndyyn 2024-09-21T17:24:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:57 smithi157 ceph-mon[25926]: mgrmap e26: smithi157.oowooq(active, since 6s), standbys: smithi018.jndyyn 2024-09-21T17:24:57.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 61397 4.29 MB/sec execute 2 sec latency 66.499 ms 2024-09-21T17:24:58.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 61811 5.87 MB/sec execute 3 sec latency 118.177 ms 2024-09-21T17:24:59.097 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:58 smithi018 ceph-mon[23738]: pgmap v6: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T17:24:59.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:58 smithi157 ceph-mon[25926]: pgmap v6: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T17:24:59.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 62364 8.97 MB/sec execute 4 sec latency 118.809 ms 2024-09-21T17:25:00.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:59 smithi157 ceph-mon[25926]: Standby manager daemon smithi018.jndyyn restarted 2024-09-21T17:25:00.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:59 smithi157 ceph-mon[25926]: Standby manager daemon smithi018.jndyyn started 2024-09-21T17:25:00.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:59 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.18:0/2148404051' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi018.jndyyn/crt"}]: dispatch 2024-09-21T17:25:00.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:59 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.18:0/2148404051' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:25:00.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:59 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.18:0/2148404051' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi018.jndyyn/key"}]: dispatch 2024-09-21T17:25:00.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:24:59 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.18:0/2148404051' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:25:00.103 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:59 smithi018 ceph-mon[23738]: Standby manager daemon smithi018.jndyyn restarted 2024-09-21T17:25:00.103 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:59 smithi018 ceph-mon[23738]: Standby manager daemon smithi018.jndyyn started 2024-09-21T17:25:00.103 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:59 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.18:0/2148404051' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi018.jndyyn/crt"}]: dispatch 2024-09-21T17:25:00.103 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:59 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.18:0/2148404051' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:25:00.103 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:59 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.18:0/2148404051' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi018.jndyyn/key"}]: dispatch 2024-09-21T17:25:00.103 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:24:59 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.18:0/2148404051' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:25:00.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 63038 9.91 MB/sec execute 5 sec latency 41.362 ms 2024-09-21T17:25:01.109 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:00 smithi018 ceph-mon[23738]: pgmap v7: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 3.4 MiB/s wr, 318 op/s 2024-09-21T17:25:01.109 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:00 smithi018 ceph-mon[23738]: mgrmap e27: smithi157.oowooq(active, since 9s), standbys: smithi018.jndyyn 2024-09-21T17:25:01.109 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:00 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:01.109 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:00 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:00 smithi157 ceph-mon[25926]: pgmap v7: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 3.4 MiB/s wr, 318 op/s 2024-09-21T17:25:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:00 smithi157 ceph-mon[25926]: mgrmap e27: smithi157.oowooq(active, since 9s), standbys: smithi018.jndyyn 2024-09-21T17:25:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:00 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:00 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:01.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 63484 11.17 MB/sec execute 6 sec latency 28.249 ms 2024-09-21T17:25:02.384 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: pgmap v8: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.7 MiB/s wr, 246 op/s 2024-09-21T17:25:02.384 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.384 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.385 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:02 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: pgmap v8: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.7 MiB/s wr, 246 op/s 2024-09-21T17:25:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:02.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:02 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:02.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 63976 11.62 MB/sec execute 7 sec latency 44.131 ms 2024-09-21T17:25:03.396 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:03 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi018 to 1937M 2024-09-21T17:25:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:03 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi018 to 1937M 2024-09-21T17:25:03.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 64406 12.33 MB/sec execute 8 sec latency 39.662 ms 2024-09-21T17:25:03.751 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:25:04.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:04 smithi018 ceph-mon[23738]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.4 MiB/s wr, 526 op/s 2024-09-21T17:25:04.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:04 smithi018 ceph-mon[23738]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:04.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 64697 11.44 MB/sec execute 9 sec latency 43.213 ms 2024-09-21T17:25:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:04 smithi157 ceph-mon[25926]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.4 MiB/s wr, 526 op/s 2024-09-21T17:25:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:04 smithi157 ceph-mon[25926]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:04.861 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:25:04.861 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (7m) 3s ago 8m 26.1M - 0.23.0 ba2b418f427c 960ba9f5795d 2024-09-21T17:25:04.861 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (8m) 3s ago 8m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:25:04.861 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (8m) 4s ago 8m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:25:04.861 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (7m) 3s ago 8m 51.4M - dad864ee21e9 4ac39470ac6e 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (2m) 3s ago 2m 35.0M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (2m) 4s ago 2m 14.4M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:9283 running (10m) 3s ago 10m 177M - 17.2.0 e1d6a67b021e 30cc894916bd 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283 running (27s) 4s ago 8m 497M - 19.3.0-5129-g0ecbb435 3fa9090ef91d cdb1e2e8af43 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (10m) 3s ago 10m 52.4M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (8m) 4s ago 8m 43.7M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (2m) 3s ago 2m 239M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (2m) 4s ago 2m 76.6M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (8m) 3s ago 8m 20.6M - 1.3.1 1dbe0e931976 a0d34619a90f 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (7m) 4s ago 7m 15.6M - 1.3.1 1dbe0e931976 cf9822488b33 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (7m) 3s ago 7m 270M 1937M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (6m) 3s ago 6m 207M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:25:04.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (6m) 3s ago 6m 176M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:25:04.863 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (5m) 3s ago 5m 208M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:25:04.863 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (5m) 4s ago 5m 235M 3984M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:25:04.863 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (5m) 4s ago 5m 256M 3984M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:25:04.863 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (4m) 4s ago 4m 241M 3984M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:25:04.863 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (4m) 4s ago 4m 328M 3984M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:25:04.863 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (7m) 3s ago 7m 65.0M - 2.33.4 514e6a882f6e 71c79331fab8 2024-09-21T17:25:05.413 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:05 smithi018 ceph-mon[23738]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:05.413 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:05 smithi018 ceph-mon[23738]: from='client.24469 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:05 smithi157 ceph-mon[25926]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:05 smithi157 ceph-mon[25926]: from='client.24469 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:05.607 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:25:05.607 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:25:05.607 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:25:05.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:25:05.608 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:25:05.738 INFO:teuthology.orchestra.run.smithi018.stdout: 5 65129 10.91 MB/sec execute 10 sec latency 51.064 ms 2024-09-21T17:25:06.232 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:25:06.232 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:25:06.232 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:25:06.232 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:25:06.232 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T17:25:06.233 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T17:25:06.233 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T17:25:06.233 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:25:06.233 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/187624981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: Adjusting osd_memory_target on smithi157 to 2704M 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: Updating smithi157:/etc/ceph/ceph.conf 2024-09-21T17:25:06.425 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:06 smithi018 ceph-mon[23738]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:25:06.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 65428 10.38 MB/sec execute 11 sec latency 153.757 ms 2024-09-21T17:25:06.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-21T17:25:06.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/187624981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:25:06.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:06.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:06.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: Adjusting osd_memory_target on smithi157 to 2704M 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: Updating smithi157:/etc/ceph/ceph.conf 2024-09-21T17:25:06.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:06 smithi157 ceph-mon[25926]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:25:06.990 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:25:07.430 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: Updating smithi157:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.431 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.432 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.432 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.432 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: Reconfiguring alertmanager.smithi018 (dependencies changed)... 2024-09-21T17:25:07.432 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: Deploying daemon alertmanager.smithi018 on smithi018 2024-09-21T17:25:07.432 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:25:07.432 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:07 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/822789939' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:25:07.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 65943 11.02 MB/sec execute 12 sec latency 42.458 ms 2024-09-21T17:25:07.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:25:07.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: Updating smithi157:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:25:07.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:25:07.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: Reconfiguring alertmanager.smithi018 (dependencies changed)... 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: Deploying daemon alertmanager.smithi018 on smithi018 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:25:07.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:07 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/822789939' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:25:08.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 66507 11.04 MB/sec execute 13 sec latency 65.628 ms 2024-09-21T17:25:09.017 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:08 smithi018 ceph-mon[23738]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-21T17:25:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:08 smithi157 ceph-mon[25926]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-21T17:25:09.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 66871 11.09 MB/sec execute 14 sec latency 33.663 ms 2024-09-21T17:25:10.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 67310 11.46 MB/sec execute 15 sec latency 60.106 ms 2024-09-21T17:25:11.040 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:10 smithi018 ceph-mon[23738]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.5 MiB/s wr, 792 op/s 2024-09-21T17:25:11.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:10 smithi157 ceph-mon[25926]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.5 MiB/s wr, 792 op/s 2024-09-21T17:25:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 67722 11.54 MB/sec execute 16 sec latency 52.765 ms 2024-09-21T17:25:12.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 68006 11.37 MB/sec execute 17 sec latency 43.677 ms 2024-09-21T17:25:13.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:12 smithi018 ceph-mon[23738]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 608 op/s 2024-09-21T17:25:13.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:12 smithi157 ceph-mon[25926]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 608 op/s 2024-09-21T17:25:13.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 68453 11.24 MB/sec execute 18 sec latency 37.844 ms 2024-09-21T17:25:14.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 68855 10.88 MB/sec execute 19 sec latency 60.847 ms 2024-09-21T17:25:15.115 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:14 smithi018 ceph-mon[23738]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 863 op/s 2024-09-21T17:25:15.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:14 smithi157 ceph-mon[25926]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 863 op/s 2024-09-21T17:25:15.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 69413 11.35 MB/sec execute 20 sec latency 31.487 ms 2024-09-21T17:25:16.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 70034 11.38 MB/sec execute 21 sec latency 41.398 ms 2024-09-21T17:25:17.188 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:16 smithi018 ceph-mon[23738]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 567 op/s 2024-09-21T17:25:17.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:16 smithi157 ceph-mon[25926]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 567 op/s 2024-09-21T17:25:17.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 70403 11.37 MB/sec execute 22 sec latency 89.776 ms 2024-09-21T17:25:18.200 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:18 smithi018 ceph-mon[23738]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 567 op/s 2024-09-21T17:25:18.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:18 smithi157 ceph-mon[25926]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 567 op/s 2024-09-21T17:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 70852 11.63 MB/sec execute 23 sec latency 43.259 ms 2024-09-21T17:25:19.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 71383 11.87 MB/sec execute 24 sec latency 51.527 ms 2024-09-21T17:25:20.235 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:20 smithi018 ceph-mon[23738]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 855 op/s 2024-09-21T17:25:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:20 smithi157 ceph-mon[25926]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 855 op/s 2024-09-21T17:25:20.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 71677 11.73 MB/sec execute 25 sec latency 42.804 ms 2024-09-21T17:25:21.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 72172 11.51 MB/sec execute 26 sec latency 39.463 ms 2024-09-21T17:25:22.279 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:22 smithi018 ceph-mon[23738]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 543 op/s 2024-09-21T17:25:22.279 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:22 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:25:22.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:22 smithi157 ceph-mon[25926]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 543 op/s 2024-09-21T17:25:22.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:22 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:25:22.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 72625 11.48 MB/sec execute 27 sec latency 52.664 ms 2024-09-21T17:25:23.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 73307 11.89 MB/sec execute 28 sec latency 32.561 ms 2024-09-21T17:25:24.739 INFO:teuthology.orchestra.run.smithi018.stdout: 5 73918 11.88 MB/sec execute 29 sec latency 55.721 ms 2024-09-21T17:25:25.052 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:24 smithi018 ceph-mon[23738]: pgmap v19: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 795 op/s 2024-09-21T17:25:25.052 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:24 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:25.052 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:24 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:25.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:24 smithi157 ceph-mon[25926]: pgmap v19: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 795 op/s 2024-09-21T17:25:25.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:24 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:25.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:24 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:25.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 74242 11.85 MB/sec execute 30 sec latency 72.500 ms 2024-09-21T17:25:26.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:25 smithi018 ceph-mon[23738]: Reconfiguring grafana.smithi018 (dependencies changed)... 2024-09-21T17:25:26.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:25 smithi018 ceph-mon[23738]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T17:25:26.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:25 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:26.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:25 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:26.059 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:25 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T17:25:26.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:25 smithi157 ceph-mon[25926]: Reconfiguring grafana.smithi018 (dependencies changed)... 2024-09-21T17:25:26.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:25 smithi157 ceph-mon[25926]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T17:25:26.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:25 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:26.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:25 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:26.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:25 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T17:25:26.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 74625 11.83 MB/sec execute 31 sec latency 50.647 ms 2024-09-21T17:25:27.066 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:26 smithi018 ceph-mon[23738]: pgmap v20: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 540 op/s 2024-09-21T17:25:27.066 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:26 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T17:25:27.066 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:26 smithi018 ceph-mon[23738]: Reconfiguring daemon grafana.smithi018 on smithi018 2024-09-21T17:25:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:26 smithi157 ceph-mon[25926]: pgmap v20: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 540 op/s 2024-09-21T17:25:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:26 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T17:25:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:26 smithi157 ceph-mon[25926]: Reconfiguring daemon grafana.smithi018 on smithi018 2024-09-21T17:25:27.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 75005 11.84 MB/sec execute 32 sec latency 82.428 ms 2024-09-21T17:25:28.070 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:28 smithi018 ceph-mon[23738]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 540 op/s 2024-09-21T17:25:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:28 smithi157 ceph-mon[25926]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 540 op/s 2024-09-21T17:25:28.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 75304 11.79 MB/sec execute 33 sec latency 72.368 ms 2024-09-21T17:25:29.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 75608 11.61 MB/sec execute 34 sec latency 45.589 ms 2024-09-21T17:25:30.356 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:30 smithi018 ceph-mon[23738]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 826 op/s 2024-09-21T17:25:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:30 smithi157 ceph-mon[25926]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 826 op/s 2024-09-21T17:25:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 75961 11.37 MB/sec execute 35 sec latency 61.827 ms 2024-09-21T17:25:31.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 76200 11.26 MB/sec execute 36 sec latency 51.921 ms 2024-09-21T17:25:32.682 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:32 smithi018 ceph-mon[23738]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.9 MiB/s wr, 537 op/s 2024-09-21T17:25:32.682 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:32 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:32.682 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:32 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:32.683 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:32 smithi018 ceph-mon[23738]: Reconfiguring node-exporter.smithi018 (dependencies changed)... 2024-09-21T17:25:32.683 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:32 smithi018 ceph-mon[23738]: Deploying daemon node-exporter.smithi018 on smithi018 2024-09-21T17:25:32.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 76400 11.15 MB/sec execute 37 sec latency 62.240 ms 2024-09-21T17:25:32.942 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:32 smithi157 ceph-mon[25926]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.9 MiB/s wr, 537 op/s 2024-09-21T17:25:32.942 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:32 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:32.943 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:32 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:32.943 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:32 smithi157 ceph-mon[25926]: Reconfiguring node-exporter.smithi018 (dependencies changed)... 2024-09-21T17:25:32.943 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:32 smithi157 ceph-mon[25926]: Deploying daemon node-exporter.smithi018 on smithi018 2024-09-21T17:25:33.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 76726 11.18 MB/sec execute 38 sec latency 56.952 ms 2024-09-21T17:25:34.726 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:34 smithi018 ceph-mon[23738]: pgmap v24: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 732 op/s 2024-09-21T17:25:34.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 77124 11.08 MB/sec execute 39 sec latency 48.566 ms 2024-09-21T17:25:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:34 smithi157 ceph-mon[25926]: pgmap v24: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 732 op/s 2024-09-21T17:25:35.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 77471 11.01 MB/sec execute 40 sec latency 63.895 ms 2024-09-21T17:25:36.706 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:36 smithi018 ceph-mon[23738]: pgmap v25: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 481 op/s 2024-09-21T17:25:36.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 77816 11.04 MB/sec execute 41 sec latency 40.539 ms 2024-09-21T17:25:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:36 smithi157 ceph-mon[25926]: pgmap v25: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 481 op/s 2024-09-21T17:25:37.541 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:25:37.732 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:37 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:25:37.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 78215 11.02 MB/sec execute 42 sec latency 59.290 ms 2024-09-21T17:25:38.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:37 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:25:38.538 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:25:38.538 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 starting - - - - 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (9m) 37s ago 9m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (8m) 37s ago 8m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 starting - - - - 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (3m) 37s ago 3m 35.0M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (3m) 37s ago 3m 14.4M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:9283 running (10m) 37s ago 10m 177M - 17.2.0 e1d6a67b021e 30cc894916bd 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283 running (60s) 37s ago 8m 497M - 19.3.0-5129-g0ecbb435 3fa9090ef91d cdb1e2e8af43 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (10m) 37s ago 10m 52.4M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (8m) 37s ago 8m 43.7M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (2m) 37s ago 2m 239M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (2m) 37s ago 2m 76.6M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (9m) 37s ago 9m 20.6M - 1.3.1 1dbe0e931976 a0d34619a90f 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (8m) 37s ago 8m 15.6M - 1.3.1 1dbe0e931976 cf9822488b33 2024-09-21T17:25:38.539 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (7m) 37s ago 7m 270M 1937M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:25:38.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (7m) 37s ago 7m 207M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:25:38.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (7m) 37s ago 7m 176M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:25:38.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (6m) 37s ago 6m 208M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:25:38.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (6m) 37s ago 6m 235M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:25:38.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (5m) 37s ago 5m 256M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:25:38.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (5m) 37s ago 5m 241M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:25:38.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (4m) 37s ago 4m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:25:38.540 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (8m) 37s ago 8m 65.0M - 2.33.4 514e6a882f6e 71c79331fab8 2024-09-21T17:25:38.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 78673 11.19 MB/sec execute 43 sec latency 73.836 ms 2024-09-21T17:25:39.009 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:38 smithi018 ceph-mon[23738]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 481 op/s 2024-09-21T17:25:39.009 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:38 smithi018 ceph-mon[23738]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:39.009 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:38 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:39.009 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:38 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:39.122 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:25:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:25:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:25:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:25:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:25:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T17:25:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:25:39.123 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:25:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:38 smithi157 ceph-mon[25926]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 481 op/s 2024-09-21T17:25:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:38 smithi157 ceph-mon[25926]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:38 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:38 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:25:39.588 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:25:39.588 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:25:39.589 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:25:39.589 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:25:39.589 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T17:25:39.589 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T17:25:39.589 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T17:25:39.589 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:25:39.589 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:25:39.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 78974 11.04 MB/sec execute 44 sec latency 53.198 ms 2024-09-21T17:25:40.025 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:39 smithi018 ceph-mon[23738]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:40.025 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:39 smithi018 ceph-mon[23738]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:40.025 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:39 smithi018 ceph-mon[23738]: Reconfiguring prometheus.smithi018 (dependencies changed)... 2024-09-21T17:25:40.025 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:39 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3128008758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:25:40.165 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:25:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:39 smithi157 ceph-mon[25926]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:39 smithi157 ceph-mon[25926]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:39 smithi157 ceph-mon[25926]: Reconfiguring prometheus.smithi018 (dependencies changed)... 2024-09-21T17:25:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:39 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3128008758' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:25:40.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 79275 10.92 MB/sec execute 45 sec latency 49.135 ms 2024-09-21T17:25:41.057 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:40 smithi018 ceph-mon[23738]: pgmap v27: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 678 op/s 2024-09-21T17:25:41.057 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:40 smithi018 ceph-mon[23738]: Deploying daemon prometheus.smithi018 on smithi018 2024-09-21T17:25:41.057 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:40 smithi018 ceph-mon[23738]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:41.057 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:40 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/851226754' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:25:41.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:40 smithi157 ceph-mon[25926]: pgmap v27: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 678 op/s 2024-09-21T17:25:41.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:40 smithi157 ceph-mon[25926]: Deploying daemon prometheus.smithi018 on smithi018 2024-09-21T17:25:41.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:40 smithi157 ceph-mon[25926]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:25:41.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:40 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/851226754' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:25:41.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 79636 10.85 MB/sec execute 46 sec latency 42.785 ms 2024-09-21T17:25:42.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 79881 10.78 MB/sec execute 47 sec latency 56.196 ms 2024-09-21T17:25:43.092 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:42 smithi018 ceph-mon[23738]: pgmap v28: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 391 op/s 2024-09-21T17:25:43.277 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:42 smithi157 ceph-mon[25926]: pgmap v28: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 391 op/s 2024-09-21T17:25:43.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 80289 10.84 MB/sec execute 48 sec latency 44.172 ms 2024-09-21T17:25:44.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 80730 10.81 MB/sec execute 49 sec latency 46.869 ms 2024-09-21T17:25:45.131 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:44 smithi018 ceph-mon[23738]: pgmap v29: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 605 op/s 2024-09-21T17:25:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:44 smithi157 ceph-mon[25926]: pgmap v29: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 605 op/s 2024-09-21T17:25:45.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 81032 10.72 MB/sec execute 50 sec latency 66.444 ms 2024-09-21T17:25:46.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:46 smithi018 ceph-mon[23738]: pgmap v30: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 410 op/s 2024-09-21T17:25:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:46 smithi157 ceph-mon[25926]: pgmap v30: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 410 op/s 2024-09-21T17:25:46.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 81346 10.73 MB/sec execute 51 sec latency 54.147 ms 2024-09-21T17:25:47.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 81739 10.72 MB/sec execute 52 sec latency 44.186 ms 2024-09-21T17:25:48.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 82106 10.80 MB/sec execute 53 sec latency 51.299 ms 2024-09-21T17:25:49.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:48 smithi018 ceph-mon[23738]: pgmap v31: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 410 op/s 2024-09-21T17:25:49.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:48 smithi157 ceph-mon[25926]: pgmap v31: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 410 op/s 2024-09-21T17:25:49.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 82380 10.73 MB/sec execute 54 sec latency 61.614 ms 2024-09-21T17:25:50.740 INFO:teuthology.orchestra.run.smithi018.stdout: 5 82795 10.66 MB/sec execute 55 sec latency 39.849 ms 2024-09-21T17:25:51.024 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:50 smithi018 ceph-mon[23738]: pgmap v32: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.3 MiB/s wr, 618 op/s 2024-09-21T17:25:51.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:50 smithi157 ceph-mon[25926]: pgmap v32: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.3 MiB/s wr, 618 op/s 2024-09-21T17:25:51.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 83283 10.67 MB/sec execute 56 sec latency 50.981 ms 2024-09-21T17:25:52.036 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:51 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:25:52.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:51 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:25:52.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 83771 10.74 MB/sec execute 57 sec latency 53.949 ms 2024-09-21T17:25:53.044 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:52 smithi018 ceph-mon[23738]: pgmap v33: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 421 op/s 2024-09-21T17:25:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:52 smithi157 ceph-mon[25926]: pgmap v33: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 421 op/s 2024-09-21T17:25:53.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 84093 10.70 MB/sec execute 58 sec latency 61.662 ms 2024-09-21T17:25:54.071 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:54 smithi018 ceph-mon[23738]: pgmap v34: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 663 op/s 2024-09-21T17:25:54.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:54 smithi157 ceph-mon[25926]: pgmap v34: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 663 op/s 2024-09-21T17:25:54.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 84330 10.61 MB/sec execute 59 sec latency 102.301 ms 2024-09-21T17:25:55.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 84588 10.53 MB/sec execute 60 sec latency 63.087 ms 2024-09-21T17:25:56.372 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:56 smithi018 ceph-mon[23738]: pgmap v35: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 449 op/s 2024-09-21T17:25:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:56 smithi157 ceph-mon[25926]: pgmap v35: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 449 op/s 2024-09-21T17:25:56.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 84839 10.52 MB/sec execute 61 sec latency 79.065 ms 2024-09-21T17:25:57.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 85061 10.44 MB/sec execute 62 sec latency 58.277 ms 2024-09-21T17:25:58.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 85373 10.46 MB/sec execute 63 sec latency 82.325 ms 2024-09-21T17:25:59.214 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:25:58 smithi018 ceph-mon[23738]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 471 op/s 2024-09-21T17:25:59.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:25:58 smithi157 ceph-mon[25926]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 471 op/s 2024-09-21T17:25:59.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 85713 10.47 MB/sec execute 64 sec latency 43.974 ms 2024-09-21T17:26:00.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 86040 10.40 MB/sec execute 65 sec latency 60.874 ms 2024-09-21T17:26:01.007 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:00 smithi018 ceph-mon[23738]: pgmap v37: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 621 op/s 2024-09-21T17:26:01.007 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:00 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:01.007 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:00 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:00 smithi157 ceph-mon[25926]: pgmap v37: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 621 op/s 2024-09-21T17:26:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:00 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:01.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:00 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:01.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 86488 10.36 MB/sec execute 66 sec latency 68.322 ms 2024-09-21T17:26:02.014 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:02 smithi018 ceph-mon[23738]: Reconfiguring node-exporter.smithi157 (dependencies changed)... 2024-09-21T17:26:02.014 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:02 smithi018 ceph-mon[23738]: Deploying daemon node-exporter.smithi157 on smithi157 2024-09-21T17:26:02.042 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:02 smithi157 ceph-mon[25926]: Reconfiguring node-exporter.smithi157 (dependencies changed)... 2024-09-21T17:26:02.042 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:02 smithi157 ceph-mon[25926]: Deploying daemon node-exporter.smithi157 on smithi157 2024-09-21T17:26:02.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 86936 10.39 MB/sec execute 67 sec latency 75.806 ms 2024-09-21T17:26:03.019 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:03 smithi018 ceph-mon[23738]: pgmap v38: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-09-21T17:26:03.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:03 smithi157 ceph-mon[25926]: pgmap v38: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-09-21T17:26:03.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 87345 10.36 MB/sec execute 68 sec latency 57.314 ms 2024-09-21T17:26:04.046 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:04 smithi018 ceph-mon[23738]: pgmap v39: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 624 op/s 2024-09-21T17:26:04.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:04 smithi157 ceph-mon[25926]: pgmap v39: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 624 op/s 2024-09-21T17:26:04.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 87761 10.43 MB/sec execute 69 sec latency 36.128 ms 2024-09-21T17:26:05.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 88126 10.44 MB/sec execute 70 sec latency 54.859 ms 2024-09-21T17:26:06.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 88378 10.41 MB/sec execute 71 sec latency 75.775 ms 2024-09-21T17:26:06.913 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: pgmap v40: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 382 op/s 2024-09-21T17:26:06.913 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:06.913 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:06.913 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi018.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi018.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi018.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi018.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:06.914 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:06.915 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:06.915 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:06 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: pgmap v40: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 382 op/s 2024-09-21T17:26:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T17:26:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T17:26:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T17:26:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi018.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi018.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi018.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi018.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:07.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:06 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:07.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 88732 10.39 MB/sec execute 72 sec latency 48.053 ms 2024-09-21T17:26:07.938 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:07 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:07.938 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:07 smithi018 ceph-mon[23738]: Upgrade: Need to upgrade myself (mgr.smithi157.oowooq) 2024-09-21T17:26:07.938 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:07 smithi018 ceph-mon[23738]: Upgrade: Need to upgrade myself (mgr.smithi157.oowooq) 2024-09-21T17:26:07.938 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:07 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:07.938 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:07 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:26:07.938 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:07 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:08.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:07 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:08.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:07 smithi157 ceph-mon[25926]: Upgrade: Need to upgrade myself (mgr.smithi157.oowooq) 2024-09-21T17:26:08.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:07 smithi157 ceph-mon[25926]: Upgrade: Need to upgrade myself (mgr.smithi157.oowooq) 2024-09-21T17:26:08.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:07 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:08.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:07 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:26:08.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:07 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:08.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 89063 10.40 MB/sec execute 73 sec latency 71.335 ms 2024-09-21T17:26:09.142 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:08 smithi018 ceph-mon[23738]: pgmap v41: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 426 op/s 2024-09-21T17:26:09.142 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:08 smithi018 ceph-mon[23738]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:09.142 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:08 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:09.142 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:08 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi018.jndyyn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:26:09.142 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:08 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi018.jndyyn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:26:09.142 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:08 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:26:09.143 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:08 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:26:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: pgmap v41: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 426 op/s 2024-09-21T17:26:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi018.jndyyn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:26:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi018.jndyyn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:26:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:26:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:26:09.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 89314 10.35 MB/sec execute 74 sec latency 64.820 ms 2024-09-21T17:26:10.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:09 smithi018 ceph-mon[23738]: Upgrade: Updating mgr.smithi018.jndyyn 2024-09-21T17:26:10.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:09 smithi018 ceph-mon[23738]: Deploying daemon mgr.smithi018.jndyyn on smithi018 2024-09-21T17:26:10.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:09 smithi018 ceph-mon[23738]: pgmap v42: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 581 op/s 2024-09-21T17:26:10.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: Upgrade: Updating mgr.smithi018.jndyyn 2024-09-21T17:26:10.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: Deploying daemon mgr.smithi018.jndyyn on smithi018 2024-09-21T17:26:10.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:09 smithi157 ceph-mon[25926]: pgmap v42: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 581 op/s 2024-09-21T17:26:10.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 89711 10.30 MB/sec execute 75 sec latency 42.662 ms 2024-09-21T17:26:10.756 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:26:11.158 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:11 smithi018 ceph-mon[23738]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:11.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:11 smithi157 ceph-mon[25926]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 90030 10.26 MB/sec execute 76 sec latency 79.214 ms 2024-09-21T17:26:11.923 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 starting - - - - 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (9m) 70s ago 9m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (9m) 71s ago 9m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 starting - - - - 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (3m) 70s ago 3m 35.0M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (3m) 71s ago 3m 14.4M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:9283 running (11m) 70s ago 11m 177M - 17.2.0 e1d6a67b021e 30cc894916bd 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283 running (94s) 71s ago 9m 497M - 19.3.0-5129-g0ecbb435 3fa9090ef91d cdb1e2e8af43 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (11m) 70s ago 11m 52.4M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (9m) 71s ago 9m 43.7M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (3m) 70s ago 3m 239M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:26:11.924 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (3m) 71s ago 3m 76.6M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 starting - - - - 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 starting - - - - 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (8m) 70s ago 8m 270M 1937M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (8m) 70s ago 8m 207M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (7m) 70s ago 7m 176M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (7m) 70s ago 7m 208M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (6m) 71s ago 6m 235M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (6m) 71s ago 6m 256M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (5m) 71s ago 5m 241M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (5m) 71s ago 5m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:26:11.925 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 starting - - - - 2024-09-21T17:26:12.525 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:26:12.525 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:26:12.525 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:26:12.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:26:12.525 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:26:12.526 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:26:12.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 90322 10.22 MB/sec execute 77 sec latency 131.729 ms 2024-09-21T17:26:13.011 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:26:13.011 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:26:13.011 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:26:13.011 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:26:13.011 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T17:26:13.011 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T17:26:13.011 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T17:26:13.012 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:26:13.012 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:26:13.202 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:12 smithi018 ceph-mon[23738]: pgmap v43: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 431 op/s 2024-09-21T17:26:13.202 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:12 smithi018 ceph-mon[23738]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:13.202 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:12 smithi018 ceph-mon[23738]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:13.202 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:12 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1535151008' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:26:13.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:12 smithi157 ceph-mon[25926]: pgmap v43: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 431 op/s 2024-09-21T17:26:13.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:12 smithi157 ceph-mon[25926]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:13.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:12 smithi157 ceph-mon[25926]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:13.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:12 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/1535151008' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:26:13.738 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:26:13.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 90632 10.20 MB/sec execute 78 sec latency 58.087 ms 2024-09-21T17:26:14.239 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:14 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/1680093028' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:26:14.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:14 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/1680093028' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:26:14.741 INFO:teuthology.orchestra.run.smithi018.stdout: 5 90936 10.13 MB/sec execute 79 sec latency 52.909 ms 2024-09-21T17:26:15.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:15 smithi018 ceph-mon[23738]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:15.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:15 smithi018 ceph-mon[23738]: pgmap v44: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.6 MiB/s wr, 637 op/s 2024-09-21T17:26:15.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:15 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:15.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:15 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:15.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:15 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:15.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:15 smithi157 ceph-mon[25926]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:15.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:15 smithi157 ceph-mon[25926]: pgmap v44: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.6 MiB/s wr, 637 op/s 2024-09-21T17:26:15.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:15 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:15.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:15 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:15.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:15 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:15.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 91240 10.14 MB/sec execute 80 sec latency 54.447 ms 2024-09-21T17:26:16.533 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:16 smithi018 ceph-mon[23738]: pgmap v45: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-09-21T17:26:16.533 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:16 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:16.533 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:16 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:16.602 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:16 smithi157 ceph-mon[25926]: pgmap v45: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-09-21T17:26:16.602 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:16 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:16.602 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:16 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:16.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 91575 10.19 MB/sec execute 81 sec latency 43.199 ms 2024-09-21T17:26:17.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 91848 10.15 MB/sec execute 82 sec latency 59.794 ms 2024-09-21T17:26:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 92128 10.10 MB/sec execute 83 sec latency 64.654 ms 2024-09-21T17:26:19.131 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:18 smithi018 ceph-mon[23738]: pgmap v46: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 450 op/s 2024-09-21T17:26:19.131 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:18 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:19.131 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:18 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:19.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:18 smithi157 ceph-mon[25926]: pgmap v46: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 450 op/s 2024-09-21T17:26:19.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:18 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:19.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:18 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:19.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 92317 10.04 MB/sec execute 84 sec latency 58.064 ms 2024-09-21T17:26:20.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 92571 10.05 MB/sec execute 85 sec latency 51.153 ms 2024-09-21T17:26:21.208 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:20 smithi018 ceph-mon[23738]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.4 MiB/s wr, 563 op/s 2024-09-21T17:26:21.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:20 smithi157 ceph-mon[25926]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.4 MiB/s wr, 563 op/s 2024-09-21T17:26:21.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 92786 10.00 MB/sec execute 86 sec latency 49.669 ms 2024-09-21T17:26:22.242 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:21 smithi018 ceph-mon[23738]: Standby manager daemon smithi018.jndyyn restarted 2024-09-21T17:26:22.242 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:21 smithi018 ceph-mon[23738]: Standby manager daemon smithi018.jndyyn started 2024-09-21T17:26:22.242 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:21 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi018.jndyyn/crt"}]: dispatch 2024-09-21T17:26:22.242 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:21 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:26:22.243 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:21 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi018.jndyyn/key"}]: dispatch 2024-09-21T17:26:22.243 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:21 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:26:22.243 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:21 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:26:22.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:21 smithi157 ceph-mon[25926]: Standby manager daemon smithi018.jndyyn restarted 2024-09-21T17:26:22.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:21 smithi157 ceph-mon[25926]: Standby manager daemon smithi018.jndyyn started 2024-09-21T17:26:22.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:21 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi018.jndyyn/crt"}]: dispatch 2024-09-21T17:26:22.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:21 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:26:22.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:21 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi018.jndyyn/key"}]: dispatch 2024-09-21T17:26:22.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:21 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:26:22.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:21 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:26:22.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 93113 9.93 MB/sec execute 87 sec latency 60.702 ms 2024-09-21T17:26:23.019 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:22 smithi018 ceph-mon[23738]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 386 op/s 2024-09-21T17:26:23.019 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:22 smithi018 ceph-mon[23738]: mgrmap e28: smithi157.oowooq(active, since 91s), standbys: smithi018.jndyyn 2024-09-21T17:26:23.019 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:22 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:23.019 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:22 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:23.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:23 smithi157 ceph-mon[25926]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 386 op/s 2024-09-21T17:26:23.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:23 smithi157 ceph-mon[25926]: mgrmap e28: smithi157.oowooq(active, since 91s), standbys: smithi018.jndyyn 2024-09-21T17:26:23.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:23 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:23.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:23 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:23.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 93446 9.92 MB/sec execute 88 sec latency 58.297 ms 2024-09-21T17:26:24.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 93660 9.85 MB/sec execute 89 sec latency 58.267 ms 2024-09-21T17:26:24.849 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:24 smithi018 ceph-mon[23738]: pgmap v49: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 538 op/s 2024-09-21T17:26:24.849 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:24 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:24.849 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:24 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:24.849 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:24 smithi018 ceph-mon[23738]: mgrmap e29: smithi157.oowooq(active, since 93s), standbys: smithi018.jndyyn 2024-09-21T17:26:25.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:24 smithi157 ceph-mon[25926]: pgmap v49: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 538 op/s 2024-09-21T17:26:25.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:24 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:25.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:24 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:25.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:24 smithi157 ceph-mon[25926]: mgrmap e29: smithi157.oowooq(active, since 93s), standbys: smithi018.jndyyn 2024-09-21T17:26:25.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 93902 9.84 MB/sec execute 90 sec latency 59.862 ms 2024-09-21T17:26:26.629 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:26 smithi018 ceph-mon[23738]: pgmap v50: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.4 MiB/s wr, 332 op/s 2024-09-21T17:26:26.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 94172 9.79 MB/sec execute 91 sec latency 66.946 ms 2024-09-21T17:26:27.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:26 smithi157 ceph-mon[25926]: pgmap v50: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.4 MiB/s wr, 332 op/s 2024-09-21T17:26:27.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 94455 9.73 MB/sec execute 92 sec latency 75.485 ms 2024-09-21T17:26:28.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 94734 9.75 MB/sec execute 93 sec latency 64.491 ms 2024-09-21T17:26:29.223 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:28 smithi018 ceph-mon[23738]: pgmap v51: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 363 op/s 2024-09-21T17:26:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:28 smithi157 ceph-mon[25926]: pgmap v51: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 363 op/s 2024-09-21T17:26:29.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 95003 9.73 MB/sec execute 94 sec latency 60.809 ms 2024-09-21T17:26:30.738 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: pgmap v52: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.4 MiB/s wr, 481 op/s 2024-09-21T17:26:30.738 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: Upgrade: Need to upgrade myself (mgr.smithi157.oowooq) 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: Upgrade: Need to upgrade myself (mgr.smithi157.oowooq) 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: Failing over to other MGR 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr fail", "who": "smithi157.oowooq"}]: dispatch 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr fail", "who": "smithi157.oowooq"}]: dispatch 2024-09-21T17:26:30.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:30 smithi018 ceph-mon[23738]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T17:26:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: 5 95423 9.79 MB/sec execute 95 sec latency 50.559 ms 2024-09-21T17:26:30.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: pgmap v52: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.4 MiB/s wr, 481 op/s 2024-09-21T17:26:30.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:30.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:30.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:26:30.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:26:30.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:30.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' 2024-09-21T17:26:30.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:30.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: Upgrade: Need to upgrade myself (mgr.smithi157.oowooq) 2024-09-21T17:26:30.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: Upgrade: Need to upgrade myself (mgr.smithi157.oowooq) 2024-09-21T17:26:30.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: Failing over to other MGR 2024-09-21T17:26:30.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 172.21.15.157:0/1708552624' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr fail", "who": "smithi157.oowooq"}]: dispatch 2024-09-21T17:26:30.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "mgr fail", "who": "smithi157.oowooq"}]: dispatch 2024-09-21T17:26:30.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:30 smithi157 ceph-mon[25926]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T17:26:31.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 95776 9.78 MB/sec execute 96 sec latency 45.518 ms 2024-09-21T17:26:31.749 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd='[{"prefix": "mgr fail", "who": "smithi157.oowooq"}]': finished 2024-09-21T17:26:31.749 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: mgrmap e30: smithi018.jndyyn(active, starting, since 0.696421s) 2024-09-21T17:26:31.749 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:26:31.749 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:26:31.749 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"}]: dispatch 2024-09-21T17:26:31.749 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T17:26:31.750 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:31 smithi018 ceph-mon[23738]: Manager daemon smithi018.jndyyn is now available 2024-09-21T17:26:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.24435 ' entity='mgr.smithi157.oowooq' cmd='[{"prefix": "mgr fail", "who": "smithi157.oowooq"}]': finished 2024-09-21T17:26:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: mgrmap e30: smithi018.jndyyn(active, starting, since 0.696421s) 2024-09-21T17:26:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:26:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:26:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"}]: dispatch 2024-09-21T17:26:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T17:26:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:31 smithi157 ceph-mon[25926]: Manager daemon smithi018.jndyyn is now available 2024-09-21T17:26:32.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 96003 9.74 MB/sec execute 97 sec latency 56.457 ms 2024-09-21T17:26:32.779 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:32 smithi018 ceph-mon[23738]: mgrmap e31: smithi018.jndyyn(active, since 1.71667s) 2024-09-21T17:26:32.780 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:32 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:32.780 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:32 smithi018 ceph-mon[23738]: pgmap v3: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:26:32.780 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:32 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:32.780 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:32 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:32.780 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:32 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:26:32.780 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:32 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:26:32.780 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:32 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:26:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:32 smithi157 ceph-mon[25926]: mgrmap e31: smithi018.jndyyn(active, since 1.71667s) 2024-09-21T17:26:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:32 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:32 smithi157 ceph-mon[25926]: pgmap v3: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:26:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:32 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:33.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:32 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:26:33.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:32 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:26:33.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:32 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"}]: dispatch 2024-09-21T17:26:33.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:32 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"}]: dispatch 2024-09-21T17:26:33.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 96249 9.71 MB/sec execute 98 sec latency 60.452 ms 2024-09-21T17:26:33.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:33 smithi157 ceph-mon[25926]: pgmap v4: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:26:33.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:33 smithi157 ceph-mon[25926]: mgrmap e32: smithi018.jndyyn(active, since 2s) 2024-09-21T17:26:33.798 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:33 smithi018 ceph-mon[23738]: pgmap v4: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:26:33.798 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:33 smithi018 ceph-mon[23738]: mgrmap e32: smithi018.jndyyn(active, since 2s) 2024-09-21T17:26:34.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 96547 9.68 MB/sec execute 99 sec latency 48.897 ms 2024-09-21T17:26:35.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 96865 9.64 MB/sec execute 100 sec latency 57.221 ms 2024-09-21T17:26:35.810 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:35 smithi018 ceph-mon[23738]: mgrmap e33: smithi018.jndyyn(active, since 4s) 2024-09-21T17:26:35.810 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:35 smithi018 ceph-mon[23738]: [21/Sep/2024:17:26:34] ENGINE Bus STARTING 2024-09-21T17:26:36.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:35 smithi157 ceph-mon[25926]: mgrmap e33: smithi018.jndyyn(active, since 4s) 2024-09-21T17:26:36.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:35 smithi157 ceph-mon[25926]: [21/Sep/2024:17:26:34] ENGINE Bus STARTING 2024-09-21T17:26:36.619 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: [21/Sep/2024:17:26:34] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T17:26:36.619 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: [21/Sep/2024:17:26:34] ENGINE Error in HTTPServer.serve 2024-09-21T17:26:36.619 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Traceback (most recent call last): 2024-09-21T17:26:36.619 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T17:26:36.619 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._connections.run(self.expiration_interval) 2024-09-21T17:26:36.619 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T17:26:36.619 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._run(expiration_interval) 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: s = self.context.wrap_socket( 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: return self.sslsocket_class._create( 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self.do_handshake() 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._sslobj.do_handshake() 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T17:26:36.620 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: [21/Sep/2024:17:26:34] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T17:26:36.621 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: [21/Sep/2024:17:26:34] ENGINE Bus STARTED 2024-09-21T17:26:36.621 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:26:36.621 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/3854433147' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:26:36.621 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/3854433147' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:26:36.621 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/3854433147' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:26:36.621 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:36 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/3854433147' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:26:36.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 97193 9.63 MB/sec execute 101 sec latency 43.874 ms 2024-09-21T17:26:36.836 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: [21/Sep/2024:17:26:34] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T17:26:36.836 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: [21/Sep/2024:17:26:34] ENGINE Error in HTTPServer.serve 2024-09-21T17:26:36.836 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Traceback (most recent call last): 2024-09-21T17:26:36.836 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._run(expiration_interval) 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self.do_handshake() 2024-09-21T17:26:36.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: [21/Sep/2024:17:26:34] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: [21/Sep/2024:17:26:34] ENGINE Bus STARTED 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/3854433147' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/3854433147' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/3854433147' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:26:36.838 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:36 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/3854433147' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:26:37.602 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:37 smithi018 ceph-mon[23738]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:26:37.602 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:37 smithi018 ceph-mon[23738]: mgrmap e34: smithi018.jndyyn(active, since 6s), standbys: smithi157.oowooq 2024-09-21T17:26:37.602 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:37 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi157.oowooq", "id": "smithi157.oowooq"}]: dispatch 2024-09-21T17:26:37.602 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:37 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:37.602 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:37 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:37.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 97498 9.63 MB/sec execute 102 sec latency 56.953 ms 2024-09-21T17:26:38.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:37 smithi157 ceph-mon[25926]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:26:38.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:37 smithi157 ceph-mon[25926]: mgrmap e34: smithi018.jndyyn(active, since 6s), standbys: smithi157.oowooq 2024-09-21T17:26:38.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:37 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr metadata", "who": "smithi157.oowooq", "id": "smithi157.oowooq"}]: dispatch 2024-09-21T17:26:38.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:37 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:38.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:37 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:38.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:38 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:38.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:38 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:38.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 97752 9.59 MB/sec execute 103 sec latency 58.591 ms 2024-09-21T17:26:39.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:38 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:39.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:38 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:39.680 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:39 smithi018 ceph-mon[23738]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.0 MiB/s wr, 330 op/s 2024-09-21T17:26:39.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 98056 9.56 MB/sec execute 104 sec latency 55.944 ms 2024-09-21T17:26:40.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:39 smithi157 ceph-mon[25926]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.0 MiB/s wr, 330 op/s 2024-09-21T17:26:40.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 98278 9.54 MB/sec execute 105 sec latency 57.606 ms 2024-09-21T17:26:41.721 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:41 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:41.721 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:41 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:41.721 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:41 smithi018 ceph-mon[23738]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.4 MiB/s wr, 256 op/s 2024-09-21T17:26:41.721 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:41 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:41.721 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:41 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:41.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 98533 9.53 MB/sec execute 106 sec latency 78.559 ms 2024-09-21T17:26:41.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:41 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:41.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:41 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:41.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:41 smithi157 ceph-mon[25926]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.4 MiB/s wr, 256 op/s 2024-09-21T17:26:41.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:41 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:41.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:41 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:42.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 98852 9.57 MB/sec execute 107 sec latency 56.370 ms 2024-09-21T17:26:43.060 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:43 smithi018 ceph-mon[23738]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.8 MiB/s wr, 386 op/s 2024-09-21T17:26:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:43 smithi157 ceph-mon[25926]: pgmap v9: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.8 MiB/s wr, 386 op/s 2024-09-21T17:26:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 99154 9.55 MB/sec execute 108 sec latency 49.729 ms 2024-09-21T17:26:44.311 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:26:44.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 99407 9.52 MB/sec execute 109 sec latency 67.048 ms 2024-09-21T17:26:45.265 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:26:45.265 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (82s) 4s ago 10m 19.9M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:26:45.265 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (10m) 4s ago 10m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:26:45.265 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (9m) 8s ago 9m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:26:45.265 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (74s) 4s ago 10m 47.8M - dad864ee21e9 48d78bd31599 2024-09-21T17:26:45.265 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (4m) 4s ago 4m 37.3M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:26:45.265 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (4m) 8s ago 4m 14.9M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (30s) 4s ago 11m 492M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283 running (2m) 8s ago 9m 165M - 19.3.0-5129-g0ecbb435 3fa9090ef91d cdb1e2e8af43 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (11m) 4s ago 11m 55.0M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (9m) 8s ago 9m 46.4M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (4m) 4s ago 4m 248M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (3m) 8s ago 3m 84.9M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (67s) 4s ago 10m 10.0M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (40s) 8s ago 9m 8006k - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (9m) 4s ago 9m 371M 1937M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (8m) 4s ago 8m 293M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (8m) 4s ago 8m 264M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (7m) 4s ago 7m 347M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (7m) 8s ago 7m 355M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (6m) 8s ago 6m 383M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:26:45.266 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (6m) 8s ago 6m 322M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:26:45.267 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (5m) 8s ago 5m 443M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:26:45.267 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (45s) 4s ago 9m 41.9M - 2.51.0 1d3b7f56885b f93b4cfd89cd 2024-09-21T17:26:45.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 99573 9.46 MB/sec execute 110 sec latency 68.607 ms 2024-09-21T17:26:45.829 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:26:45.829 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:26:45.829 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:26:45.829 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:26:45.829 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:26:45.829 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:26:45.829 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:26:45.830 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:26:45.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:45 smithi018 ceph-mon[23738]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:45.837 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:45 smithi018 ceph-mon[23738]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.4 MiB/s wr, 351 op/s 2024-09-21T17:26:46.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:45 smithi157 ceph-mon[25926]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:46.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:45 smithi157 ceph-mon[25926]: pgmap v10: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.4 MiB/s wr, 351 op/s 2024-09-21T17:26:46.319 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:26:46.319 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:26:46.319 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:26:46.320 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:26:46.320 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:26:46.320 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr" 2024-09-21T17:26:46.320 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:26:46.320 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "2/23 daemons upgraded", 2024-09-21T17:26:46.320 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T17:26:46.320 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:26:46.320 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:26:46.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 99740 9.40 MB/sec execute 111 sec latency 49.395 ms 2024-09-21T17:26:46.856 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:46 smithi018 ceph-mon[23738]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:46.857 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:46 smithi018 ceph-mon[23738]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:46.857 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:46 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/4080941515' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:26:46.898 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:26:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:46 smithi157 ceph-mon[25926]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:46 smithi157 ceph-mon[25926]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:46 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/4080941515' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:26:47.743 INFO:teuthology.orchestra.run.smithi018.stdout: 5 99974 9.39 MB/sec execute 112 sec latency 88.345 ms 2024-09-21T17:26:47.771 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:47.771 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.4 MiB/s wr, 351 op/s 2024-09-21T17:26:47.771 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:26:47.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3766934608' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:26:47.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:47.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:47.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:26:47.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:26:47.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:47.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:47 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:48.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:26:48.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.4 MiB/s wr, 351 op/s 2024-09-21T17:26:48.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:26:48.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3766934608' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:26:48.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:48.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:48.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:26:48.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:26:48.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:48.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:47 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:48.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 100190 9.33 MB/sec execute 113 sec latency 64.012 ms 2024-09-21T17:26:48.787 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:26:48.787 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: Updating smithi157:/etc/ceph/ceph.conf 2024-09-21T17:26:48.787 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:26:48.787 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:26:48.787 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:26:48.787 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: Updating smithi157:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:26:48.788 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:26:48.788 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:26:48.788 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:48.788 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:48.788 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:48.788 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:48 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:26:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: Updating smithi157:/etc/ceph/ceph.conf 2024-09-21T17:26:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:26:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:26:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:26:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: Updating smithi157:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:26:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:26:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:26:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:48 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:49.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 100446 9.31 MB/sec execute 114 sec latency 67.633 ms 2024-09-21T17:26:49.847 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:49 smithi018 ceph-mon[23738]: Reconfiguring prometheus.smithi018 (dependencies changed)... 2024-09-21T17:26:49.847 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:49 smithi018 ceph-mon[23738]: Reconfiguring daemon prometheus.smithi018 on smithi018 2024-09-21T17:26:49.847 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:49 smithi018 ceph-mon[23738]: pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.1 MiB/s wr, 507 op/s 2024-09-21T17:26:50.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:49 smithi157 ceph-mon[25926]: Reconfiguring prometheus.smithi018 (dependencies changed)... 2024-09-21T17:26:50.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:49 smithi157 ceph-mon[25926]: Reconfiguring daemon prometheus.smithi018 on smithi018 2024-09-21T17:26:50.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:49 smithi157 ceph-mon[25926]: pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.1 MiB/s wr, 507 op/s 2024-09-21T17:26:50.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 100710 9.28 MB/sec execute 115 sec latency 76.422 ms 2024-09-21T17:26:51.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 101029 9.29 MB/sec execute 116 sec latency 65.896 ms 2024-09-21T17:26:51.893 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:51 smithi018 ceph-mon[23738]: pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 317 op/s 2024-09-21T17:26:52.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:51 smithi157 ceph-mon[25926]: pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 317 op/s 2024-09-21T17:26:52.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 101298 9.26 MB/sec execute 117 sec latency 62.051 ms 2024-09-21T17:26:53.183 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:53 smithi018 ceph-mon[23738]: pgmap v14: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-09-21T17:26:53.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:53 smithi157 ceph-mon[25926]: pgmap v14: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-09-21T17:26:53.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 101665 9.27 MB/sec execute 118 sec latency 52.279 ms 2024-09-21T17:26:54.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 101923 9.24 MB/sec execute 119 sec latency 70.575 ms 2024-09-21T17:26:55.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 102209 9.26 MB/sec execute 120 sec latency 73.443 ms 2024-09-21T17:26:55.775 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:55 smithi018 ceph-mon[23738]: pgmap v15: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 312 op/s 2024-09-21T17:26:56.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:55 smithi157 ceph-mon[25926]: pgmap v15: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 312 op/s 2024-09-21T17:26:56.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 102500 9.27 MB/sec execute 121 sec latency 67.241 ms 2024-09-21T17:26:57.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 102856 9.25 MB/sec execute 122 sec latency 48.263 ms 2024-09-21T17:26:57.805 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:57 smithi018 ceph-mon[23738]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 312 op/s 2024-09-21T17:26:57.805 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:57 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:57.805 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:57 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:57.805 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:57 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:26:58.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:57 smithi157 ceph-mon[25926]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 312 op/s 2024-09-21T17:26:58.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:57 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:58.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:57 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:26:58.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:57 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:26:58.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 103152 9.25 MB/sec execute 123 sec latency 79.236 ms 2024-09-21T17:26:58.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:58 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:26:58.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:58 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:58.853 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:58 smithi018 ceph-mon[23738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T17:26:58.853 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:58 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:26:59.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 103407 9.22 MB/sec execute 124 sec latency 44.244 ms 2024-09-21T17:26:59.872 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:59 smithi018 ceph-mon[23738]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 484 op/s 2024-09-21T17:26:59.872 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:59 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:26:59.872 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:59 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:26:59.872 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:26:59 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:26:59.940 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:59 smithi157 ceph-mon[25926]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 484 op/s 2024-09-21T17:26:59.940 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:59 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T17:26:59.940 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:59 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T17:26:59.940 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:26:59 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:27:00.635 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:00 smithi018 ceph-mon[23738]: Upgrade: Updating mgr.smithi157.oowooq 2024-09-21T17:27:00.635 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:00 smithi018 ceph-mon[23738]: Deploying daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:27:00.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 103776 9.20 MB/sec execute 125 sec latency 41.775 ms 2024-09-21T17:27:00.897 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:00 smithi157 ceph-mon[25926]: Upgrade: Updating mgr.smithi157.oowooq 2024-09-21T17:27:00.897 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:00 smithi157 ceph-mon[25926]: Deploying daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:27:01.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:01 smithi018 ceph-mon[23738]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.3 MiB/s wr, 327 op/s 2024-09-21T17:27:01.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 104086 9.19 MB/sec execute 126 sec latency 45.007 ms 2024-09-21T17:27:01.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:01 smithi157 ceph-mon[25926]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.3 MiB/s wr, 327 op/s 2024-09-21T17:27:02.669 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:02 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:27:02.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 104463 9.20 MB/sec execute 127 sec latency 68.002 ms 2024-09-21T17:27:02.913 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:02 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:27:03.687 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:03 smithi018 ceph-mon[23738]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 523 op/s 2024-09-21T17:27:03.687 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:03 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:03.687 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:03 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:03.687 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:03 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:27:03.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 104755 9.21 MB/sec execute 128 sec latency 61.959 ms 2024-09-21T17:27:04.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:03 smithi157 ceph-mon[25926]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 523 op/s 2024-09-21T17:27:04.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:03 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:04.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:03 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:04.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:03 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:27:04.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 105196 9.21 MB/sec execute 129 sec latency 73.936 ms 2024-09-21T17:27:04.958 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:04 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:04.958 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:04 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:04.991 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:04 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:04.991 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:04 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:05.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 105482 9.20 MB/sec execute 130 sec latency 53.911 ms 2024-09-21T17:27:05.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:05 smithi157 ceph-mon[25926]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 366 op/s 2024-09-21T17:27:05.989 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:05 smithi018 ceph-mon[23738]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 366 op/s 2024-09-21T17:27:06.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 105817 9.24 MB/sec execute 131 sec latency 73.513 ms 2024-09-21T17:27:07.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 106133 9.24 MB/sec execute 132 sec latency 48.870 ms 2024-09-21T17:27:07.761 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:07 smithi018 ceph-mon[23738]: pgmap v21: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 366 op/s 2024-09-21T17:27:07.799 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:07 smithi157 ceph-mon[25926]: pgmap v21: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 366 op/s 2024-09-21T17:27:08.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 106428 9.23 MB/sec execute 133 sec latency 46.081 ms 2024-09-21T17:27:09.294 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:09 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:09.294 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:09 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:09.294 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:09 smithi018 ceph-mon[23738]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.9 MiB/s wr, 570 op/s 2024-09-21T17:27:09.294 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:09 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:09.294 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:09 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:09.462 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:09 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:09.462 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:09 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:09.462 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:09 smithi157 ceph-mon[25926]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.9 MiB/s wr, 570 op/s 2024-09-21T17:27:09.462 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:09 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:09.462 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:09 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:09.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 106659 9.19 MB/sec execute 134 sec latency 75.562 ms 2024-09-21T17:27:10.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:10 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/2471873230' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:27:10.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:10 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/2471873230' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:27:10.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:10 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/2471873230' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:27:10.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:10 smithi018 ceph-mon[23738]: from='mgr.? 172.21.15.157:0/2471873230' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:27:10.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:10 smithi018 ceph-mon[23738]: Standby manager daemon smithi157.oowooq restarted 2024-09-21T17:27:10.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:10 smithi018 ceph-mon[23738]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:27:10.421 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:10 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/2471873230' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:27:10.421 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:10 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/2471873230' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:27:10.421 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:10 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/2471873230' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:27:10.422 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:10 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/2471873230' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:27:10.422 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:10 smithi157 ceph-mon[25926]: Standby manager daemon smithi157.oowooq restarted 2024-09-21T17:27:10.422 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:10 smithi157 ceph-mon[25926]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:27:10.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 106865 9.16 MB/sec execute 135 sec latency 76.347 ms 2024-09-21T17:27:11.339 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:11 smithi018 ceph-mon[23738]: mgrmap e35: smithi018.jndyyn(active, since 40s), standbys: smithi157.oowooq 2024-09-21T17:27:11.339 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:11 smithi018 ceph-mon[23738]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 398 op/s 2024-09-21T17:27:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:11 smithi157 ceph-mon[25926]: mgrmap e35: smithi018.jndyyn(active, since 40s), standbys: smithi157.oowooq 2024-09-21T17:27:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:11 smithi157 ceph-mon[25926]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 398 op/s 2024-09-21T17:27:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: 5 107212 9.14 MB/sec execute 136 sec latency 47.780 ms 2024-09-21T17:27:12.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 107574 9.14 MB/sec execute 137 sec latency 43.794 ms 2024-09-21T17:27:12.879 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:12 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:12.879 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:12 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:12.879 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:12 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:12.879 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:12 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:12.879 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:12 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:27:12.879 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:12 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:27:12.879 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:12 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:12.880 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:12 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:12 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:12 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:12 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:12 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:12 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:27:13.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:12 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T17:27:13.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:12 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:13.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:12 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:13.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 107923 9.13 MB/sec execute 138 sec latency 44.944 ms 2024-09-21T17:27:13.902 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 568 op/s 2024-09-21T17:27:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:27:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:27:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi018.jndyyn"}]: dispatch 2024-09-21T17:27:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi018.jndyyn"}]': finished 2024-09-21T17:27:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi157.oowooq"}]: dispatch 2024-09-21T17:27:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi157.oowooq"}]': finished 2024-09-21T17:27:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:13 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 568 op/s 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi018.jndyyn"}]: dispatch 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi018.jndyyn"}]': finished 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi157.oowooq"}]: dispatch 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi157.oowooq"}]': finished 2024-09-21T17:27:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:13 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T17:27:14.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 108297 9.16 MB/sec execute 139 sec latency 46.674 ms 2024-09-21T17:27:14.921 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:14 smithi018 ceph-mon[23738]: Upgrade: Setting container_image for all mgr 2024-09-21T17:27:15.072 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:14 smithi157 ceph-mon[25926]: Upgrade: Setting container_image for all mgr 2024-09-21T17:27:15.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 108732 9.18 MB/sec execute 140 sec latency 65.426 ms 2024-09-21T17:27:15.946 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:15 smithi018 ceph-mon[23738]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-21T17:27:15.946 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:15 smithi018 ceph-mon[23738]: Upgrade: Updating mon.smithi018 2024-09-21T17:27:15.946 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:15 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:15.946 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:15 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T17:27:15.946 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:15 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T17:27:15.946 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:15 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:27:15.947 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:15 smithi018 ceph-mon[23738]: Deploying daemon mon.smithi018 on smithi018 2024-09-21T17:27:16.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:15 smithi157 ceph-mon[25926]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-21T17:27:16.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:15 smithi157 ceph-mon[25926]: Upgrade: Updating mon.smithi018 2024-09-21T17:27:16.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:15 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:16.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:15 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T17:27:16.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:15 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T17:27:16.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:15 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T17:27:16.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:15 smithi157 ceph-mon[25926]: Deploying daemon mon.smithi018 on smithi018 2024-09-21T17:27:16.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 108980 9.15 MB/sec execute 141 sec latency 47.742 ms 2024-09-21T17:27:16.987 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:16 smithi018 ceph-mon[23738]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-21T17:27:16.988 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:16 smithi018 ceph-mon[23738]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:27:17.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:16 smithi157 ceph-mon[25926]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-21T17:27:17.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:16 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T17:27:17.364 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:27:17.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 109322 9.19 MB/sec execute 142 sec latency 44.046 ms 2024-09-21T17:27:18.008 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:17 smithi018 ceph-mon[23738]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:18.246 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (115s) 6s ago 11m 20.0M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (11m) 6s ago 11m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (10m) 10s ago 10m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (107s) 6s ago 10m 46.6M - dad864ee21e9 48d78bd31599 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (4m) 6s ago 4m 37.2M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (4m) 10s ago 4m 15.0M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (63s) 6s ago 12m 509M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (15s) 10s ago 10m 88.0M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (12m) 6s ago 12m 55.3M 2048M 17.2.0 e1d6a67b021e 405cef9a3f40 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (10m) 10s ago 10m 45.7M 2048M 17.2.0 e1d6a67b021e 37814dd0c3ac 2024-09-21T17:27:18.247 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (4m) 6s ago 4m 244M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (4m) 10s ago 4m 86.5M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (100s) 6s ago 10m 10.1M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (73s) 10s ago 10m 8182k - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (9m) 6s ago 9m 413M 1937M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (9m) 6s ago 9m 325M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (8m) 6s ago 8m 269M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (8m) 6s ago 8m 389M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (7m) 10s ago 7m 376M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (7m) 10s ago 7m 400M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (6m) 10s ago 6m 342M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (6m) 10s ago 6m 478M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:27:18.248 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (21s) 6s ago 10m 39.4M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:27:18.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:17 smithi157 ceph-mon[25926]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:18.527 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:18 smithi018 systemd[1]: Stopping Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:27:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 109582 9.17 MB/sec execute 143 sec latency 66.979 ms 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:27:18.828 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:27:18.829 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:27:18.829 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:27:18.829 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:27:18.829 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:27:18.829 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T17:27:18.829 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:27:18.829 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:27:18.829 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:27:19.042 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 ceph-mon[23738]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:19.043 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 ceph-mon[23738]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:19.043 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 ceph-mon[23738]: pgmap v27: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.0 MiB/s wr, 589 op/s 2024-09-21T17:27:19.043 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 ceph-mon[23738]: from='client.? 172.21.15.18:0/3533400168' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:27:19.043 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[23734]: 2024-09-21T17:27:19.023+0000 7f37f5cba700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi018 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:27:19.043 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[23734]: 2024-09-21T17:27:19.023+0000 7f37f5cba700 -1 mon.smithi018@0(leader) e2 *** Got Signal Terminated *** 2024-09-21T17:27:19.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:19 smithi157 ceph-mon[25926]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:19.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:19 smithi157 ceph-mon[25926]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:19.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:19 smithi157 ceph-mon[25926]: pgmap v27: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.0 MiB/s wr, 589 op/s 2024-09-21T17:27:19.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:19 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/3533400168' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T17:27:19.294 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:27:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:27:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:27:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:27:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:27:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr" 2024-09-21T17:27:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:27:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "2/23 daemons upgraded", 2024-09-21T17:27:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T17:27:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:27:19.295 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:27:19.295 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 podman[63715]: 2024-09-21 17:27:19.069092429 +0000 UTC m=+0.349071273 container died 405cef9a3f40cbe8e0248758cccde1cc7f6cdd2ad26795d083c1090fdd697926 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, name=centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., RELEASE=HEAD) 2024-09-21T17:27:19.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 109828 9.18 MB/sec execute 144 sec latency 82.214 ms 2024-09-21T17:27:19.885 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:27:20.060 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 podman[63715]: 2024-09-21 17:27:19.82005166 +0000 UTC m=+1.100030488 container cleanup 405cef9a3f40cbe8e0248758cccde1cc7f6cdd2ad26795d083c1090fdd697926 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, ceph=True, vendor=Red Hat, Inc., name=centos-stream, version=8) 2024-09-21T17:27:20.060 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:19 smithi018 bash[63715]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018 2024-09-21T17:27:20.311 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:20 smithi018 podman[63749]: 2024-09-21 17:27:20.167960039 +0000 UTC m=+1.096483663 container remove 405cef9a3f40cbe8e0248758cccde1cc7f6cdd2ad26795d083c1090fdd697926 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, RELEASE=HEAD, distribution-scope=public, vcs-type=git, version=8, io.buildah.version=1.19.8, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc.) 2024-09-21T17:27:20.568 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:20 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018.service: Deactivated successfully. 2024-09-21T17:27:20.568 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:20 smithi018 systemd[1]: Stopped Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:27:20.568 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:20 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018.service: Consumed 17.139s CPU time. 2024-09-21T17:27:20.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 110027 9.13 MB/sec execute 145 sec latency 60.841 ms 2024-09-21T17:27:20.836 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:20 smithi018 systemd[1]: Starting Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:27:21.102 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 podman[63906]: 2024-09-21 17:27:20.969726574 +0000 UTC m=+0.020873488 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:27:21.370 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 podman[63906]: 2024-09-21 17:27:21.100600367 +0000 UTC m=+0.151747269 container create d7cfe2f8a92d77056c5a5814d0de87eecd3188e23d079e4422a3bcdea9282fa6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T17:27:21.638 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 podman[63906]: 2024-09-21 17:27:21.503570453 +0000 UTC m=+0.554717368 container init d7cfe2f8a92d77056c5a5814d0de87eecd3188e23d079e4422a3bcdea9282fa6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 podman[63906]: 2024-09-21 17:27:21.514741226 +0000 UTC m=+0.565888130 container start d7cfe2f8a92d77056c5a5814d0de87eecd3188e23d079e4422a3bcdea9282fa6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev), process ceph-mon, pid 2 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: pidfile_write: ignore empty --pid-file 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: load: jerasure load: lrc 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: RocksDB version: 7.9.2 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Git sha 0 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Compile date 2024-09-21 14:38:11 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: DB SUMMARY 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: DB Session ID: 0S2R7UZD1LIC0H5725DH 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: CURRENT file: CURRENT 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T17:27:21.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi018/store.db dir, Total Num: 1, files: 000030.sst 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi018/store.db: 000028.log size: 2460720 ; 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.error_if_exists: 0 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.create_if_missing: 0 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.paranoid_checks: 1 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.env: 0x5571e7673120 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.info_log: 0x5571e99278e0 2024-09-21T17:27:21.640 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T17:27:21.641 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.statistics: (nil) 2024-09-21T17:27:21.641 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.use_fsync: 0 2024-09-21T17:27:21.641 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_log_file_size: 0 2024-09-21T17:27:21.641 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T17:27:21.641 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T17:27:21.641 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T17:27:21.641 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T17:27:21.641 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.allow_fallocate: 1 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.use_direct_reads: 0 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.db_log_dir: 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.wal_dir: 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T17:27:21.642 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.write_buffer_manager: 0x5571e992bb80 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.unordered_write: 0 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T17:27:21.643 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.row_cache: None 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.wal_filter: None 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.two_write_queues: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.wal_compression: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.atomic_flush: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T17:27:21.644 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.log_readahead_size: 0 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_background_jobs: 2 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_background_compactions: -1 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_subcompactions: 1 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T17:27:21.645 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_open_files: -1 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_background_flushes: -1 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Compression algorithms supported: 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: kZSTD supported: 0 2024-09-21T17:27:21.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: kXpressCompression supported: 0 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: kBZip2Compression supported: 0 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: kLZ4Compression supported: 1 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: kZlibCompression supported: 1 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: kSnappyCompression supported: 1 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi018/store.db/MANIFEST-000015 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.merge_operator: 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_filter: None 2024-09-21T17:27:21.647 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5571e9927540) 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: cache_index_and_filter_blocks: 1 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: pin_top_level_index_and_filter: 1 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: index_type: 0 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: data_block_index_type: 0 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: index_shortening: 1 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: checksum: 4 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: no_block_cache: 0 2024-09-21T17:27:21.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_cache: 0x5571e991b610 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_cache_name: BinnedLRUCache 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_cache_options: 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: capacity : 536870912 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: num_shard_bits : 4 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: strict_capacity_limit : 0 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: high_pri_pool_ratio: 0.000 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_cache_compressed: (nil) 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: persistent_cache: (nil) 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_size: 4096 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_size_deviation: 10 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_restart_interval: 16 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: index_block_restart_interval: 1 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: metadata_block_size: 4096 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: partition_filters: 0 2024-09-21T17:27:21.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: use_delta_encoding: 1 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: filter_policy: bloomfilter 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: whole_key_filtering: 1 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: verify_compression: 0 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: read_amp_bytes_per_bit: 0 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: format_version: 5 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: enable_index_compression: 1 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: block_align: 0 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: max_auto_readahead_size: 262144 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: prepopulate_block_cache: 0 2024-09-21T17:27:21.650 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: initial_auto_readahead_size: 8192 2024-09-21T17:27:21.651 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T17:27:21.651 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T17:27:21.651 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T17:27:21.651 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression: NoCompression 2024-09-21T17:27:21.651 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T17:27:21.651 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.num_levels: 7 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T17:27:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T17:27:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T17:27:21.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T17:27:21.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.inplace_update_support: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.bloom_locality: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.max_successive_merges: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T17:27:21.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.ttl: 2592000 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.enable_blob_files: false 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.min_blob_size: 0 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T17:27:21.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi018/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11844, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 822a755b-4721-463f-bf1c-76daf2f8bed8 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726939641548332, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726939641558206, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2250654, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11845, "largest_seqno": 12870, "table_properties": {"data_size": 2245359, "index_size": 3098, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10291, "raw_average_key_size": 23, "raw_value_size": 2235647, "raw_average_value_size": 5023, "num_data_blocks": 141, "num_entries": 445, "num_filter_entries": 445, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726939641, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "822a755b-4721-463f-bf1c-76daf2f8bed8", "db_session_id": "0S2R7UZD1LIC0H5725DH", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726939641558352, "job": 1, "event": "recovery_finished"} 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi018/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5571e994ae00 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: DB pointer 0x5571e9a4c000 2024-09-21T17:27:21.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** DB Stats ** 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T17:27:21.659 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: L0 1/0 2.15 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 493.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: L6 1/0 7.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Sum 2/0 9.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 493.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 493.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** Compaction Stats [default] ** 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 493.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-21T17:27:21.660 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Cumulative compaction: 0.00 GB write, 144.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Interval compaction: 0.00 GB write, 144.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Block cache BinnedLRUCache@0x5571e991b610#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: starting mon.smithi018 rank 0 at public addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] at bind addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi018 fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: mon.smithi018@-1(???) e2 preinit fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: mon.smithi018@-1(???).mds e7 new map 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: mon.smithi018@-1(???).mds e7 print_map 2024-09-21T17:27:21.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: e7 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: legacy client fscid: 1 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Filesystem 'foofs' (1) 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: fs_name foofs 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: epoch 5 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: created 2024-09-21T17:22:16.251763+0000 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: modified 2024-09-21T17:22:22.574871+0000 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: tableserver 0 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: root 0 2024-09-21T17:27:21.662 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: session_timeout 60 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: session_autoclose 300 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: max_file_size 1099511627776 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: max_xattr_size 65536 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: required_client_features {} 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: last_failure 0 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: last_failure_osd_epoch 0 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: max_mds 1 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: in 0 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: up {0=14496} 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: failed 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: damaged 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: stopped 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: data_pools [3] 2024-09-21T17:27:21.663 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: metadata_pool 2 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: inline_data disabled 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: balancer 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: bal_rank_mask -1 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: standby_count_wanted 1 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: qdb_cluster leader: 0 members: 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: [mds.foofs.smithi018.wzitfj{0:14496} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.18:6834/1237994936,v1:172.21.15.18:6835/1237994936] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Standby daemons: 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: [mds.foofs.smithi157.iijaxo{-1:24291} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: mon.smithi018@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: mon.smithi018@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: mon.smithi018@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T17:27:21.664 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: mon.smithi018@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T17:27:21.665 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 ceph-mon[63924]: mon.smithi018@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-21T17:27:21.665 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 bash[63906]: d7cfe2f8a92d77056c5a5814d0de87eecd3188e23d079e4422a3bcdea9282fa6 2024-09-21T17:27:21.665 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:21 smithi018 systemd[1]: Started Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:27:21.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 110222 9.09 MB/sec execute 146 sec latency 67.150 ms 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1404169393' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: pgmap v28: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 386 op/s 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: mon.smithi018 calling monitor election 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: mon.smithi018 is new leader, mons smithi018,smithi157 in quorum (ranks 0,1) 2024-09-21T17:27:22.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: monmap epoch 2 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: last_changed 2024-09-21T17:16:56.579645+0000 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: created 2024-09-21T17:14:43.780044+0000 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: min_mon_release 17 (quincy) 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: election_strategy: 1 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.smithi018 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: 1: [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] mon.smithi157 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: mgrmap e35: smithi018.jndyyn(active, since 52s), standbys: smithi157.oowooq 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: overall HEALTH_OK 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: from='mgr.14780 ' entity='' 2024-09-21T17:27:22.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:22 smithi018 ceph-mon[63924]: mgrmap e36: smithi018.jndyyn(active, since 52s), standbys: smithi157.oowooq 2024-09-21T17:27:22.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi018"}]: dispatch 2024-09-21T17:27:22.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "mon metadata", "id": "smithi157"}]: dispatch 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: from='mgr.14780 172.21.15.18:0/1424527392' entity='mgr.smithi018.jndyyn' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: from='client.? 172.21.15.18:0/1404169393' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: pgmap v28: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 386 op/s 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: mon.smithi018 calling monitor election 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: mon.smithi018 is new leader, mons smithi018,smithi157 in quorum (ranks 0,1) 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: monmap epoch 2 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: last_changed 2024-09-21T17:16:56.579645+0000 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: created 2024-09-21T17:14:43.780044+0000 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: min_mon_release 17 (quincy) 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: election_strategy: 1 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.smithi018 2024-09-21T17:27:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: 1: [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] mon.smithi157 2024-09-21T17:27:22.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:27:22.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T17:27:22.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: mgrmap e35: smithi018.jndyyn(active, since 52s), standbys: smithi157.oowooq 2024-09-21T17:27:22.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: overall HEALTH_OK 2024-09-21T17:27:22.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: from='mgr.14780 ' entity='' 2024-09-21T17:27:22.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:22 smithi157 ceph-mon[25926]: mgrmap e36: smithi018.jndyyn(active, since 52s), standbys: smithi157.oowooq 2024-09-21T17:27:22.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 110392 9.07 MB/sec execute 147 sec latency 60.320 ms 2024-09-21T17:27:23.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 110796 9.05 MB/sec execute 148 sec latency 45.863 ms 2024-09-21T17:27:24.221 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:24 smithi018 ceph-mon[63924]: mgrmap e37: smithi018.jndyyn(active, since 53s), standbys: smithi157.oowooq 2024-09-21T17:27:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:24 smithi157 ceph-mon[25926]: mgrmap e37: smithi018.jndyyn(active, since 53s), standbys: smithi157.oowooq 2024-09-21T17:27:24.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 111140 9.05 MB/sec execute 149 sec latency 43.239 ms 2024-09-21T17:27:25.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 111447 9.04 MB/sec execute 150 sec latency 36.687 ms 2024-09-21T17:27:26.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 111809 9.06 MB/sec execute 151 sec latency 49.881 ms 2024-09-21T17:27:27.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 112163 9.04 MB/sec execute 152 sec latency 39.256 ms 2024-09-21T17:27:28.290 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:28 smithi018 ceph-mon[63924]: Standby manager daemon smithi157.oowooq restarted 2024-09-21T17:27:28.290 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:28 smithi018 ceph-mon[63924]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:27:28.290 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:28 smithi018 ceph-mon[63924]: from='mgr.? 172.21.15.157:0/398456448' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:27:28.290 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:28 smithi018 ceph-mon[63924]: from='mgr.? 172.21.15.157:0/398456448' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:27:28.291 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:28 smithi018 ceph-mon[63924]: from='mgr.? 172.21.15.157:0/398456448' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:27:28.291 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:28 smithi018 ceph-mon[63924]: from='mgr.? 172.21.15.157:0/398456448' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:27:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:28 smithi157 ceph-mon[25926]: Standby manager daemon smithi157.oowooq restarted 2024-09-21T17:27:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:28 smithi157 ceph-mon[25926]: Standby manager daemon smithi157.oowooq started 2024-09-21T17:27:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:28 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/398456448' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/crt"}]: dispatch 2024-09-21T17:27:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:28 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/398456448' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T17:27:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:28 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/398456448' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi157.oowooq/key"}]: dispatch 2024-09-21T17:27:28.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:28 smithi157 ceph-mon[25926]: from='mgr.? 172.21.15.157:0/398456448' entity='mgr.smithi157.oowooq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T17:27:28.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 112476 9.05 MB/sec execute 153 sec latency 38.144 ms 2024-09-21T17:27:29.311 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:29 smithi018 ceph-mon[63924]: mgrmap e38: smithi018.jndyyn(active, since 58s), standbys: smithi157.oowooq 2024-09-21T17:27:29.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:29 smithi157 ceph-mon[25926]: mgrmap e38: smithi018.jndyyn(active, since 58s), standbys: smithi157.oowooq 2024-09-21T17:27:29.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 112849 9.09 MB/sec execute 154 sec latency 34.736 ms 2024-09-21T17:27:30.344 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: Active manager daemon smithi018.jndyyn restarted 2024-09-21T17:27:30.344 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: Activating manager daemon smithi018.jndyyn 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: mgrmap e39: smithi018.jndyyn(active, starting, since 0.00668874s), standbys: smithi157.oowooq 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata", "id": "smithi018"} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata", "id": "smithi157"} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mgr metadata", "who": "smithi157.oowooq", "id": "smithi157.oowooq"} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T17:27:30.345 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T17:27:30.346 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T17:27:30.346 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T17:27:30.346 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T17:27:30.346 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T17:27:30.346 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T17:27:30.346 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T17:27:30.346 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:30 smithi018 ceph-mon[63924]: Manager daemon smithi018.jndyyn is now available 2024-09-21T17:27:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: Active manager daemon smithi018.jndyyn restarted 2024-09-21T17:27:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: Activating manager daemon smithi018.jndyyn 2024-09-21T17:27:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: mgrmap e39: smithi018.jndyyn(active, starting, since 0.00668874s), standbys: smithi157.oowooq 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata", "id": "smithi018"} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata", "id": "smithi157"} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mgr metadata", "who": "smithi018.jndyyn", "id": "smithi018.jndyyn"} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mgr metadata", "who": "smithi157.oowooq", "id": "smithi157.oowooq"} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T17:27:30.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T17:27:30.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T17:27:30.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T17:27:30.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T17:27:30.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T17:27:30.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:30 smithi157 ceph-mon[25926]: Manager daemon smithi018.jndyyn is now available 2024-09-21T17:27:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 113188 9.10 MB/sec execute 155 sec latency 60.888 ms 2024-09-21T17:27:31.193 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:31 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:27:31.193 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:31 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"} : dispatch 2024-09-21T17:27:31.193 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:31 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:27:31.193 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:31 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T17:27:31.193 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:31 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"} : dispatch 2024-09-21T17:27:31.193 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:31 smithi157 ceph-mon[25926]: mgrmap e40: smithi018.jndyyn(active, since 1.02146s), standbys: smithi157.oowooq 2024-09-21T17:27:31.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:27:31.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/mirror_snapshot_schedule"} : dispatch 2024-09-21T17:27:31.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:27:31.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T17:27:31.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi018.jndyyn/trash_purge_schedule"} : dispatch 2024-09-21T17:27:31.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:31 smithi018 ceph-mon[63924]: mgrmap e40: smithi018.jndyyn(active, since 1.02146s), standbys: smithi157.oowooq 2024-09-21T17:27:31.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 113475 9.09 MB/sec execute 156 sec latency 64.279 ms 2024-09-21T17:27:32.170 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:32 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:27:32.170 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:32 smithi018 ceph-mon[63924]: pgmap v3: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:27:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:32 smithi157 ceph-mon[25926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T17:27:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:32 smithi157 ceph-mon[25926]: pgmap v3: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:27:32.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 113761 9.07 MB/sec execute 157 sec latency 61.387 ms 2024-09-21T17:27:33.182 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:33 smithi018 ceph-mon[63924]: mgrmap e41: smithi018.jndyyn(active, since 2s), standbys: smithi157.oowooq 2024-09-21T17:27:33.182 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:33 smithi018 ceph-mon[63924]: [21/Sep/2024:17:27:32] ENGINE Bus STARTING 2024-09-21T17:27:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:33 smithi157 ceph-mon[25926]: mgrmap e41: smithi018.jndyyn(active, since 2s), standbys: smithi157.oowooq 2024-09-21T17:27:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:33 smithi157 ceph-mon[25926]: [21/Sep/2024:17:27:32] ENGINE Bus STARTING 2024-09-21T17:27:33.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 114125 9.05 MB/sec execute 158 sec latency 47.618 ms 2024-09-21T17:27:34.200 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:34 smithi018 ceph-mon[63924]: [21/Sep/2024:17:27:32] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T17:27:34.200 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:34 smithi018 ceph-mon[63924]: [21/Sep/2024:17:27:32] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T17:27:34.200 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:34 smithi018 ceph-mon[63924]: [21/Sep/2024:17:27:32] ENGINE Bus STARTED 2024-09-21T17:27:34.200 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:34 smithi018 ceph-mon[63924]: [21/Sep/2024:17:27:32] ENGINE Error in HTTPServer.serve 2024-09-21T17:27:34.200 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: Traceback (most recent call last): 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._connections.run(self.expiration_interval) 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._run(expiration_interval) 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: s = self.context.wrap_socket( 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: return self.sslsocket_class._create( 2024-09-21T17:27:34.201 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T17:27:34.202 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self.do_handshake() 2024-09-21T17:27:34.202 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T17:27:34.202 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: self._sslobj.do_handshake() 2024-09-21T17:27:34.202 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T17:27:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:34 smithi157 ceph-mon[25926]: [21/Sep/2024:17:27:32] ENGINE Serving on http://172.21.15.18:8765 2024-09-21T17:27:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:34 smithi157 ceph-mon[25926]: [21/Sep/2024:17:27:32] ENGINE Serving on https://172.21.15.18:7150 2024-09-21T17:27:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:34 smithi157 ceph-mon[25926]: [21/Sep/2024:17:27:32] ENGINE Bus STARTED 2024-09-21T17:27:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:34 smithi157 ceph-mon[25926]: [21/Sep/2024:17:27:32] ENGINE Error in HTTPServer.serve 2024-09-21T17:27:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Traceback (most recent call last): 2024-09-21T17:27:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._connections.run(self.expiration_interval) 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._run(expiration_interval) 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: s = self.context.wrap_socket( 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: return self.sslsocket_class._create( 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self.do_handshake() 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T17:27:34.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: self._sslobj.do_handshake() 2024-09-21T17:27:34.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T17:27:34.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 114574 9.07 MB/sec execute 159 sec latency 31.404 ms 2024-09-21T17:27:35.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:35 smithi018 ceph-mon[63924]: pgmap v5: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:27:35.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:35 smithi018 ceph-mon[63924]: mgrmap e42: smithi018.jndyyn(active, since 4s), standbys: smithi157.oowooq 2024-09-21T17:27:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:35 smithi157 ceph-mon[25926]: pgmap v5: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:27:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:35 smithi157 ceph-mon[25926]: mgrmap e42: smithi018.jndyyn(active, since 4s), standbys: smithi157.oowooq 2024-09-21T17:27:35.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 115030 9.09 MB/sec execute 160 sec latency 31.222 ms 2024-09-21T17:27:36.468 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:36 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:36.468 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:36 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:36.468 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:36 smithi018 ceph-mon[63924]: pgmap v6: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:27:36.468 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:36 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:36.468 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:36 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:36.691 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:36 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:36.691 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:36 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:36.691 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:36 smithi157 ceph-mon[25926]: pgmap v6: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T17:27:36.691 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:36 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:36.691 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:36 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:36.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 115506 9.11 MB/sec execute 161 sec latency 58.143 ms 2024-09-21T17:27:37.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 115961 9.13 MB/sec execute 162 sec latency 44.968 ms 2024-09-21T17:27:38.745 INFO:teuthology.orchestra.run.smithi018.stdout: 5 116217 9.12 MB/sec execute 163 sec latency 82.012 ms 2024-09-21T17:27:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:39 smithi157 ceph-mon[25926]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 3.7 MiB/s wr, 412 op/s 2024-09-21T17:27:39.281 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:39 smithi018 ceph-mon[63924]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 3.7 MiB/s wr, 412 op/s 2024-09-21T17:27:39.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 116465 9.14 MB/sec execute 164 sec latency 65.202 ms 2024-09-21T17:27:40.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.9 MiB/s wr, 320 op/s 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:27:40.555 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:40 smithi018 ceph-mon[63924]: Updating smithi157:/etc/ceph/ceph.conf 2024-09-21T17:27:40.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 116878 9.15 MB/sec execute 165 sec latency 40.693 ms 2024-09-21T17:27:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.9 MiB/s wr, 320 op/s 2024-09-21T17:27:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:40.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:27:40.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:27:40.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: Updating smithi018:/etc/ceph/ceph.conf 2024-09-21T17:27:40.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:40 smithi157 ceph-mon[25926]: Updating smithi157:/etc/ceph/ceph.conf 2024-09-21T17:27:41.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 117312 9.15 MB/sec execute 166 sec latency 32.491 ms 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: Updating smithi157:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.076 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.077 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.077 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:27:42.077 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:27:42.077 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T17:27:42.242 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:27:42.242 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T17:27:42.242 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: Updating smithi157:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:27:42.242 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2024-09-21T17:27:42.242 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: Updating smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:27:42.242 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: Updating smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T17:27:42.242 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.242 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.243 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.243 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.243 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.243 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:42.243 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:27:42.243 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:27:42.243 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:41 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T17:27:42.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 117779 9.16 MB/sec execute 167 sec latency 67.442 ms 2024-09-21T17:27:43.111 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:42 smithi018 ceph-mon[63924]: pgmap v9: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.9 MiB/s wr, 498 op/s 2024-09-21T17:27:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:42 smithi157 ceph-mon[25926]: pgmap v9: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.9 MiB/s wr, 498 op/s 2024-09-21T17:27:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 118102 9.13 MB/sec execute 168 sec latency 47.016 ms 2024-09-21T17:27:44.164 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:44 smithi018 ceph-mon[63924]: Upgrade: Updating mon.smithi157 2024-09-21T17:27:44.164 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:44.164 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T17:27:44.164 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T17:27:44.165 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:27:44.165 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:44 smithi018 ceph-mon[63924]: Deploying daemon mon.smithi157 on smithi157 2024-09-21T17:27:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:44 smithi157 ceph-mon[25926]: Upgrade: Updating mon.smithi157 2024-09-21T17:27:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:44 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:44 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T17:27:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:44 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T17:27:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:44 smithi157 ceph-mon[25926]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:27:44.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:44 smithi157 ceph-mon[25926]: Deploying daemon mon.smithi157 on smithi157 2024-09-21T17:27:44.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 118355 9.12 MB/sec execute 169 sec latency 76.442 ms 2024-09-21T17:27:45.151 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:45 smithi157 ceph-mon[25926]: pgmap v10: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.5 MiB/s wr, 456 op/s 2024-09-21T17:27:45.175 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:45 smithi018 ceph-mon[63924]: pgmap v10: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.5 MiB/s wr, 456 op/s 2024-09-21T17:27:45.447 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:45 smithi157 systemd[1]: Stopping Ceph mon.smithi157 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:27:45.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 118801 9.16 MB/sec execute 170 sec latency 66.787 ms 2024-09-21T17:27:45.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:45 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157[25904]: 2024-09-21T17:27:45.444+0000 7fb723b83700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi157 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:27:45.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:45 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157[25904]: 2024-09-21T17:27:45.444+0000 7fb723b83700 -1 mon.smithi157@1(peon) e2 *** Got Signal Terminated *** 2024-09-21T17:27:45.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:45 smithi157 podman[49305]: 2024-09-21 17:27:45.465265388 +0000 UTC m=+0.151234469 container died 37814dd0c3aca816ff19f30e1c64b3f671aadec52ebeb2bb11d13894fe38cdf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, release=754, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., name=centos-stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, version=8, distribution-scope=public, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-21T17:27:46.140 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:45 smithi157 podman[49305]: 2024-09-21 17:27:45.903205857 +0000 UTC m=+0.589174942 container cleanup 37814dd0c3aca816ff19f30e1c64b3f671aadec52ebeb2bb11d13894fe38cdf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, io.buildah.version=1.19.8, GIT_CLEAN=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream) 2024-09-21T17:27:46.140 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:45 smithi157 bash[49305]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157 2024-09-21T17:27:46.395 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:46 smithi157 podman[49317]: 2024-09-21 17:27:46.138607406 +0000 UTC m=+0.673536555 container remove 37814dd0c3aca816ff19f30e1c64b3f671aadec52ebeb2bb11d13894fe38cdf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, version=8, architecture=x86_64, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, distribution-scope=public, ceph=True) 2024-09-21T17:27:46.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 119189 9.14 MB/sec execute 171 sec latency 45.201 ms 2024-09-21T17:27:46.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:46 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi157.service: Deactivated successfully. 2024-09-21T17:27:46.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:46 smithi157 systemd[1]: Stopped Ceph mon.smithi157 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:27:46.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:46 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi157.service: Consumed 6.861s CPU time. 2024-09-21T17:27:46.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:46 smithi157 systemd[1]: Starting Ceph mon.smithi157 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:27:47.103 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:46 smithi157 podman[49442]: 2024-09-21 17:27:46.737531983 +0000 UTC m=+0.017183081 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:27:47.103 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:46 smithi157 podman[49442]: 2024-09-21 17:27:46.848313948 +0000 UTC m=+0.127965030 container create 1c6557eec7d04ac5b25070e9e3d683f453969b02cd981f925565a600ef42e208 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T17:27:47.103 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 podman[49442]: 2024-09-21 17:27:47.101568843 +0000 UTC m=+0.381219932 container init 1c6557eec7d04ac5b25070e9e3d683f453969b02cd981f925565a600ef42e208 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:27:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 podman[49442]: 2024-09-21 17:27:47.106084267 +0000 UTC m=+0.385735348 container start 1c6557eec7d04ac5b25070e9e3d683f453969b02cd981f925565a600ef42e208 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T17:27:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev), process ceph-mon, pid 2 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: pidfile_write: ignore empty --pid-file 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: load: jerasure load: lrc 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: RocksDB version: 7.9.2 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Git sha 0 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Compile date 2024-09-21 14:38:11 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: DB SUMMARY 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: DB Session ID: CR49BUFMET29TMMST9HC 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: CURRENT file: CURRENT 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T17:27:47.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi157/store.db dir, Total Num: 1, files: 000024.sst 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi157/store.db: 000022.log size: 7207381 ; 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.error_if_exists: 0 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.create_if_missing: 0 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.paranoid_checks: 1 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.env: 0x563a80ffa120 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.info_log: 0x563a830cf8e0 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T17:27:47.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.statistics: (nil) 2024-09-21T17:27:47.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.use_fsync: 0 2024-09-21T17:27:47.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_log_file_size: 0 2024-09-21T17:27:47.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T17:27:47.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T17:27:47.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T17:27:47.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T17:27:47.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.allow_fallocate: 1 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.use_direct_reads: 0 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.db_log_dir: 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.wal_dir: 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T17:27:47.533 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.write_buffer_manager: 0x563a830d3b80 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.unordered_write: 0 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T17:27:47.534 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.row_cache: None 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.wal_filter: None 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.two_write_queues: 0 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T17:27:47.535 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.wal_compression: 0 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.atomic_flush: 0 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.log_readahead_size: 0 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T17:27:47.536 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_background_jobs: 2 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_background_compactions: -1 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_subcompactions: 1 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_open_files: -1 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T17:27:47.537 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_background_flushes: -1 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Compression algorithms supported: 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: kZSTD supported: 0 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: kXpressCompression supported: 0 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: kBZip2Compression supported: 0 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: kLZ4Compression supported: 1 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: kZlibCompression supported: 1 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T17:27:47.538 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: kSnappyCompression supported: 1 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi157/store.db/MANIFEST-000009 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.merge_operator: 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_filter: None 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563a830cf540) 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: cache_index_and_filter_blocks: 1 2024-09-21T17:27:47.539 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: pin_top_level_index_and_filter: 1 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: index_type: 0 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: data_block_index_type: 0 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: index_shortening: 1 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: checksum: 4 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: no_block_cache: 0 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: block_cache: 0x563a830c3610 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: block_cache_name: BinnedLRUCache 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: block_cache_options: 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: capacity : 536870912 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: num_shard_bits : 4 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: strict_capacity_limit : 0 2024-09-21T17:27:47.540 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: high_pri_pool_ratio: 0.000 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: block_cache_compressed: (nil) 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: persistent_cache: (nil) 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: block_size: 4096 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: block_size_deviation: 10 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: block_restart_interval: 16 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: index_block_restart_interval: 1 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: metadata_block_size: 4096 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: partition_filters: 0 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: use_delta_encoding: 1 2024-09-21T17:27:47.541 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: filter_policy: bloomfilter 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: whole_key_filtering: 1 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: verify_compression: 0 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: read_amp_bytes_per_bit: 0 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: format_version: 5 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: enable_index_compression: 1 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: block_align: 0 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: max_auto_readahead_size: 262144 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: prepopulate_block_cache: 0 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: initial_auto_readahead_size: 8192 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T17:27:47.542 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression: NoCompression 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.num_levels: 7 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T17:27:47.543 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T17:27:47.544 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T17:27:47.545 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T17:27:47.546 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.inplace_update_support: 0 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.bloom_locality: 0 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.max_successive_merges: 0 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T17:27:47.547 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.ttl: 2592000 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.enable_blob_files: false 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.min_blob_size: 0 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T17:27:47.548 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi157/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12254, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 47c397cc-e8de-4c72-bae9-e70bb2242c47 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726939667135378, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726939667151873, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4373440, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12259, "largest_seqno": 13932, "table_properties": {"data_size": 4365922, "index_size": 4745, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15505, "raw_average_key_size": 23, "raw_value_size": 4351405, "raw_average_value_size": 6494, "num_data_blocks": 215, "num_entries": 670, "num_filter_entries": 670, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726939667, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "47c397cc-e8de-4c72-bae9-e70bb2242c47", "db_session_id": "CR49BUFMET29TMMST9HC", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726939667152022, "job": 1, "event": "recovery_finished"} 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi157/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563a830f2e00 2024-09-21T17:27:47.549 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: DB pointer 0x563a831f4000 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: ** DB Stats ** 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: ** Compaction Stats [default] ** 2024-09-21T17:27:47.550 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: L0 1/0 4.17 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 626.6 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: L6 1/0 7.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Sum 2/0 11.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 626.6 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 626.6 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: ** Compaction Stats [default] ** 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 626.6 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.551 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T17:27:47.552 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-21T17:27:47.552 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T17:27:47.552 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T17:27:47.552 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T17:27:47.552 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T17:27:47.552 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Cumulative compaction: 0.00 GB write, 212.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T17:27:47.552 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Interval compaction: 0.00 GB write, 212.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Block cache BinnedLRUCache@0x563a830c3610#2 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: starting mon.smithi157 rank 1 at public addrs [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] at bind addrs [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi157 fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: mon.smithi157@-1(???) e2 preinit fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: mon.smithi157@-1(???).mds e7 new map 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: mon.smithi157@-1(???).mds e7 print_map 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: e7 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T17:27:47.553 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: legacy client fscid: 1 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Filesystem 'foofs' (1) 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: fs_name foofs 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: epoch 5 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: created 2024-09-21T17:22:16.251763+0000 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: modified 2024-09-21T17:22:22.574871+0000 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: tableserver 0 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: root 0 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: session_timeout 60 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: session_autoclose 300 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: max_file_size 1099511627776 2024-09-21T17:27:47.554 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: max_xattr_size 65536 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: required_client_features {} 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: last_failure 0 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: last_failure_osd_epoch 0 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: max_mds 1 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: in 0 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: up {0=14496} 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: failed 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: damaged 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: stopped 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: data_pools [3] 2024-09-21T17:27:47.555 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: metadata_pool 2 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: inline_data disabled 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: balancer 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: bal_rank_mask -1 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: standby_count_wanted 1 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: qdb_cluster leader: 0 members: 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: [mds.foofs.smithi018.wzitfj{0:14496} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.18:6834/1237994936,v1:172.21.15.18:6835/1237994936] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: Standby daemons: 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: 2024-09-21T17:27:47.556 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout: [mds.foofs.smithi157.iijaxo{-1:24291} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T17:27:47.557 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: mon.smithi157@-1(???).osd e74 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T17:27:47.557 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: mon.smithi157@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T17:27:47.557 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: mon.smithi157@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T17:27:47.557 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: mon.smithi157@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T17:27:47.557 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 ceph-mon[49460]: mon.smithi157@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-21T17:27:47.557 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 bash[49442]: 1c6557eec7d04ac5b25070e9e3d683f453969b02cd981f925565a600ef42e208 2024-09-21T17:27:47.557 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:47 smithi157 systemd[1]: Started Ceph mon.smithi157 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:27:47.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 119539 9.16 MB/sec execute 172 sec latency 47.361 ms 2024-09-21T17:27:48.506 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: 1: [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] mon.smithi157 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: mgrmap e42: smithi018.jndyyn(active, since 17s), standbys: smithi157.oowooq 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: overall HEALTH_OK 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: mon.smithi018 calling monitor election 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: mon.smithi157 calling monitor election 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: mon.smithi018 is new leader, mons smithi018,smithi157 in quorum (ranks 0,1) 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata", "id": "smithi018"} : dispatch 2024-09-21T17:27:48.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: monmap epoch 3 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: last_changed 2024-09-21T17:27:47.467453+0000 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: created 2024-09-21T17:14:43.780044+0000 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: min_mon_release 19 (squid) 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: election_strategy: 1 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.smithi018 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: 1: [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] mon.smithi157 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: mgrmap e42: smithi018.jndyyn(active, since 17s), standbys: smithi157.oowooq 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: overall HEALTH_OK 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata", "id": "smithi157"} : dispatch 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:48.508 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:27:48.509 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:48 smithi018 ceph-mon[63924]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 6.7 MiB/s wr, 691 op/s 2024-09-21T17:27:48.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 119923 9.19 MB/sec execute 173 sec latency 55.285 ms 2024-09-21T17:27:48.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: 1: [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] mon.smithi157 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: mgrmap e42: smithi018.jndyyn(active, since 17s), standbys: smithi157.oowooq 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: overall HEALTH_OK 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: mon.smithi018 calling monitor election 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: mon.smithi157 calling monitor election 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: mon.smithi018 is new leader, mons smithi018,smithi157 in quorum (ranks 0,1) 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata", "id": "smithi018"} : dispatch 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: monmap epoch 3 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: fsid ec44ef70-783c-11ef-baf6-efdc52797490 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: last_changed 2024-09-21T17:27:47.467453+0000 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: created 2024-09-21T17:14:43.780044+0000 2024-09-21T17:27:48.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: min_mon_release 19 (squid) 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: election_strategy: 1 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: 0: [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon.smithi018 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: 1: [v2:172.21.15.157:3300/0,v1:172.21.15.157:6789/0] mon.smithi157 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: mgrmap e42: smithi018.jndyyn(active, since 17s), standbys: smithi157.oowooq 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: overall HEALTH_OK 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mon metadata", "id": "smithi157"} : dispatch 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:27:48.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:48 smithi157 ceph-mon[49460]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 6.7 MiB/s wr, 691 op/s 2024-09-21T17:27:49.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 120317 9.21 MB/sec execute 174 sec latency 54.316 ms 2024-09-21T17:27:50.377 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:27:50.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 120625 9.20 MB/sec execute 175 sec latency 50.183 ms 2024-09-21T17:27:51.092 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:51 smithi018 ceph-mon[63924]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 451 op/s 2024-09-21T17:27:51.256 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:27:51.256 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (2m) 11s ago 11m 20.0M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:27:51.256 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (11m) 11s ago 11m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:27:51.256 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (11m) 15s ago 11m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:27:51.256 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (2m) 11s ago 11m 46.3M - dad864ee21e9 48d78bd31599 2024-09-21T17:27:51.256 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (5m) 11s ago 5m 37.3M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:27:51.256 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (5m) 15s ago 5m 15.4M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:27:51.256 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (96s) 11s ago 12m 489M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (48s) 15s ago 10m 454M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (30s) 11s ago 13m 30.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 starting - - - 2048M 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (5m) 11s ago 5m 261M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (5m) 15s ago 5m 86.6M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (2m) 11s ago 11m 9.99M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (106s) 15s ago 10m 12.0M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (10m) 11s ago 10m 446M 1937M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (9m) 11s ago 9m 352M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (9m) 11s ago 9m 294M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (8m) 11s ago 8m 401M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (8m) 15s ago 8m 387M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (7m) 15s ago 7m 438M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:27:51.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (7m) 15s ago 7m 371M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:27:51.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (7m) 15s ago 7m 504M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:27:51.258 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (54s) 11s ago 10m 46.4M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:27:51.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:51 smithi157 ceph-mon[49460]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 451 op/s 2024-09-21T17:27:51.744 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:27:51.744 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:27:51.744 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:27:51.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:27:51.744 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:27:51.744 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:27:51.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:27:51.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 4 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:27:51.745 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:27:51.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 120783 9.17 MB/sec execute 176 sec latency 77.282 ms 2024-09-21T17:27:52.107 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:52 smithi018 ceph-mon[63924]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:52.107 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:52 smithi018 ceph-mon[63924]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:52.107 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:52 smithi018 ceph-mon[63924]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:52.107 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:52 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2665864314' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:27:52.107 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:52 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:52.107 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:52 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:52.227 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:27:52.227 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:27:52.227 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:27:52.227 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:27:52.227 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:27:52.227 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:27:52.228 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:27:52.228 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:27:52.228 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "4/23 daemons upgraded", 2024-09-21T17:27:52.228 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T17:27:52.228 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:27:52.228 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:27:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:52 smithi157 ceph-mon[49460]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:52 smithi157 ceph-mon[49460]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:52 smithi157 ceph-mon[49460]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:52 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2665864314' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:27:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:52 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:52 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:52.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 120968 9.13 MB/sec execute 177 sec latency 70.367 ms 2024-09-21T17:27:52.761 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:27:53.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:53 smithi018 ceph-mon[63924]: pgmap v14: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 663 op/s 2024-09-21T17:27:53.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:53 smithi018 ceph-mon[63924]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:53.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:53.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:53.400 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:53 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2474976937' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:27:53.465 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:53 smithi157 ceph-mon[49460]: pgmap v14: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 663 op/s 2024-09-21T17:27:53.465 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:53 smithi157 ceph-mon[49460]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:27:53.465 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:53.465 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:53.465 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:53 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2474976937' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:27:53.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 121185 9.11 MB/sec execute 178 sec latency 62.594 ms 2024-09-21T17:27:54.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 121440 9.09 MB/sec execute 179 sec latency 92.704 ms 2024-09-21T17:27:55.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:55 smithi018 ceph-mon[63924]: pgmap v15: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-21T17:27:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:55 smithi157 ceph-mon[49460]: pgmap v15: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-21T17:27:55.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 121725 9.08 MB/sec execute 180 sec latency 80.291 ms 2024-09-21T17:27:56.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 122031 9.07 MB/sec execute 181 sec latency 45.283 ms 2024-09-21T17:27:57.301 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: pgmap v16: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-21T17:27:57.301 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:57.301 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:57.301 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:27:57.301 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:27:57.301 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:57.302 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:57.302 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T17:27:57.302 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T17:27:57.302 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:57 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:27:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: pgmap v16: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-21T17:27:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:27:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:27:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:57.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:27:57.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T17:27:57.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T17:27:57.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:57 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:27:57.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 122435 9.10 MB/sec execute 182 sec latency 47.288 ms 2024-09-21T17:27:58.402 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:58 smithi018 ceph-mon[63924]: Reconfiguring mon.smithi018 (monmap changed)... 2024-09-21T17:27:58.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:58 smithi018 ceph-mon[63924]: Reconfiguring daemon mon.smithi018 on smithi018 2024-09-21T17:27:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:58 smithi157 ceph-mon[49460]: Reconfiguring mon.smithi018 (monmap changed)... 2024-09-21T17:27:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:58 smithi157 ceph-mon[49460]: Reconfiguring daemon mon.smithi018 on smithi018 2024-09-21T17:27:58.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 122813 9.07 MB/sec execute 183 sec latency 98.796 ms 2024-09-21T17:27:59.419 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:27:59 smithi018 ceph-mon[63924]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.0 MiB/s wr, 594 op/s 2024-09-21T17:27:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:27:59 smithi157 ceph-mon[49460]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.0 MiB/s wr, 594 op/s 2024-09-21T17:27:59.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 123093 9.08 MB/sec execute 184 sec latency 78.070 ms 2024-09-21T17:28:00.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:00.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:00.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:00 smithi018 ceph-mon[63924]: Reconfiguring mgr.smithi018.jndyyn (monmap changed)... 2024-09-21T17:28:00.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi018.jndyyn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T17:28:00.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T17:28:00.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:00.204 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:00 smithi018 ceph-mon[63924]: Reconfiguring daemon mgr.smithi018.jndyyn on smithi018 2024-09-21T17:28:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:00 smithi157 ceph-mon[49460]: Reconfiguring mgr.smithi018.jndyyn (monmap changed)... 2024-09-21T17:28:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi018.jndyyn", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T17:28:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T17:28:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:00 smithi157 ceph-mon[49460]: Reconfiguring daemon mgr.smithi018.jndyyn on smithi018 2024-09-21T17:28:00.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 123878 9.18 MB/sec execute 185 sec latency 48.873 ms 2024-09-21T17:28:01.221 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:01 smithi018 ceph-mon[63924]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 358 op/s 2024-09-21T17:28:01.221 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:28:01.221 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:01.222 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:01.222 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi018", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T17:28:01.222 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:01 smithi157 ceph-mon[49460]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 358 op/s 2024-09-21T17:28:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:28:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi018", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T17:28:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:01.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 124325 9.19 MB/sec execute 186 sec latency 38.514 ms 2024-09-21T17:28:02.244 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:02 smithi018 ceph-mon[63924]: Reconfiguring crash.smithi018 (monmap changed)... 2024-09-21T17:28:02.244 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:02 smithi018 ceph-mon[63924]: Reconfiguring daemon crash.smithi018 on smithi018 2024-09-21T17:28:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:02 smithi157 ceph-mon[49460]: Reconfiguring crash.smithi018 (monmap changed)... 2024-09-21T17:28:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:02 smithi157 ceph-mon[49460]: Reconfiguring daemon crash.smithi018 on smithi018 2024-09-21T17:28:02.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 124592 9.17 MB/sec execute 187 sec latency 40.152 ms 2024-09-21T17:28:03.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:03 smithi018 ceph-mon[63924]: pgmap v19: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.5 MiB/s wr, 616 op/s 2024-09-21T17:28:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:03 smithi157 ceph-mon[49460]: pgmap v19: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.5 MiB/s wr, 616 op/s 2024-09-21T17:28:03.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 124948 9.16 MB/sec execute 188 sec latency 80.274 ms 2024-09-21T17:28:04.531 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:04.532 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:04.532 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:04 smithi018 ceph-mon[63924]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T17:28:04.532 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T17:28:04.532 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:04.532 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:04 smithi018 ceph-mon[63924]: Reconfiguring daemon osd.0 on smithi018 2024-09-21T17:28:04.532 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:04 smithi018 ceph-mon[63924]: pgmap v20: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 405 op/s 2024-09-21T17:28:04.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 125239 9.15 MB/sec execute 189 sec latency 95.842 ms 2024-09-21T17:28:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:04 smithi157 ceph-mon[49460]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T17:28:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T17:28:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:04 smithi157 ceph-mon[49460]: Reconfiguring daemon osd.0 on smithi018 2024-09-21T17:28:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:04 smithi157 ceph-mon[49460]: pgmap v20: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 405 op/s 2024-09-21T17:28:05.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 125472 9.13 MB/sec execute 190 sec latency 77.305 ms 2024-09-21T17:28:06.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 125840 9.15 MB/sec execute 191 sec latency 61.334 ms 2024-09-21T17:28:06.831 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:06.831 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:06.831 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:06 smithi018 ceph-mon[63924]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T17:28:06.831 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T17:28:06.831 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:06.831 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:06 smithi018 ceph-mon[63924]: Reconfiguring daemon osd.1 on smithi018 2024-09-21T17:28:06.832 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:06 smithi018 ceph-mon[63924]: pgmap v21: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 405 op/s 2024-09-21T17:28:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:06 smithi157 ceph-mon[49460]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T17:28:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T17:28:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:06 smithi157 ceph-mon[49460]: Reconfiguring daemon osd.1 on smithi018 2024-09-21T17:28:07.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:06 smithi157 ceph-mon[49460]: pgmap v21: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 405 op/s 2024-09-21T17:28:07.746 INFO:teuthology.orchestra.run.smithi018.stdout: 5 126172 9.13 MB/sec execute 192 sec latency 73.588 ms 2024-09-21T17:28:08.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 126565 9.14 MB/sec execute 193 sec latency 56.426 ms 2024-09-21T17:28:09.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:09 smithi018 ceph-mon[63924]: pgmap v22: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.0 MiB/s wr, 599 op/s 2024-09-21T17:28:09.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:09 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:09.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:09 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:09.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:09 smithi018 ceph-mon[63924]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T17:28:09.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:09 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T17:28:09.137 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:09 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:09.137 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:09 smithi018 ceph-mon[63924]: Reconfiguring daemon osd.2 on smithi018 2024-09-21T17:28:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:09 smithi157 ceph-mon[49460]: pgmap v22: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.0 MiB/s wr, 599 op/s 2024-09-21T17:28:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:09 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:09 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:09 smithi157 ceph-mon[49460]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T17:28:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:09 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T17:28:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:09 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:09 smithi157 ceph-mon[49460]: Reconfiguring daemon osd.2 on smithi018 2024-09-21T17:28:09.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 126823 9.13 MB/sec execute 194 sec latency 64.598 ms 2024-09-21T17:28:10.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 127152 9.16 MB/sec execute 195 sec latency 69.863 ms 2024-09-21T17:28:11.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:11 smithi018 ceph-mon[63924]: pgmap v23: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 452 op/s 2024-09-21T17:28:11.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:11 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:11.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:11 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:11.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:11 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T17:28:11.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:11 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:11 smithi157 ceph-mon[49460]: pgmap v23: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 452 op/s 2024-09-21T17:28:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:11 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:11 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:11 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T17:28:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:11 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 127531 9.17 MB/sec execute 196 sec latency 77.869 ms 2024-09-21T17:28:12.257 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:12 smithi018 ceph-mon[63924]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T17:28:12.257 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:12 smithi018 ceph-mon[63924]: Reconfiguring daemon osd.3 on smithi018 2024-09-21T17:28:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:12 smithi157 ceph-mon[49460]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T17:28:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:12 smithi157 ceph-mon[49460]: Reconfiguring daemon osd.3 on smithi018 2024-09-21T17:28:12.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 127879 9.16 MB/sec execute 197 sec latency 43.739 ms 2024-09-21T17:28:13.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:13 smithi018 ceph-mon[63924]: pgmap v24: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.8 MiB/s wr, 652 op/s 2024-09-21T17:28:13.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:13 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:13.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:13 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:13.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:13 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi018.wzitfj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T17:28:13.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:13 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:13 smithi157 ceph-mon[49460]: pgmap v24: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.8 MiB/s wr, 652 op/s 2024-09-21T17:28:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:13 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:13 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:13 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi018.wzitfj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T17:28:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:13 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:13.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 128161 9.15 MB/sec execute 198 sec latency 54.528 ms 2024-09-21T17:28:14.290 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:14 smithi018 ceph-mon[63924]: Reconfiguring mds.foofs.smithi018.wzitfj (monmap changed)... 2024-09-21T17:28:14.290 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:14 smithi018 ceph-mon[63924]: Reconfiguring daemon mds.foofs.smithi018.wzitfj on smithi018 2024-09-21T17:28:14.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:14 smithi157 ceph-mon[49460]: Reconfiguring mds.foofs.smithi018.wzitfj (monmap changed)... 2024-09-21T17:28:14.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:14 smithi157 ceph-mon[49460]: Reconfiguring daemon mds.foofs.smithi018.wzitfj on smithi018 2024-09-21T17:28:14.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 128491 9.13 MB/sec execute 199 sec latency 110.137 ms 2024-09-21T17:28:15.305 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:15 smithi018 ceph-mon[63924]: pgmap v25: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 393 op/s 2024-09-21T17:28:15.305 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:15 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:15.305 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:15 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:15.305 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:15 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi157", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T17:28:15.305 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:15 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:15.385 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:15 smithi157 ceph-mon[49460]: pgmap v25: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 393 op/s 2024-09-21T17:28:15.385 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:15 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:15.385 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:15 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:15.385 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:15 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi157", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T17:28:15.385 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:15 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:15.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 128765 9.12 MB/sec execute 200 sec latency 53.701 ms 2024-09-21T17:28:16.342 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:16 smithi018 ceph-mon[63924]: Reconfiguring crash.smithi157 (monmap changed)... 2024-09-21T17:28:16.343 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:16 smithi018 ceph-mon[63924]: Reconfiguring daemon crash.smithi157 on smithi157 2024-09-21T17:28:16.343 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:28:16.389 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:16 smithi157 ceph-mon[49460]: Reconfiguring crash.smithi157 (monmap changed)... 2024-09-21T17:28:16.389 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:16 smithi157 ceph-mon[49460]: Reconfiguring daemon crash.smithi157 on smithi157 2024-09-21T17:28:16.389 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:28:16.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 129026 9.10 MB/sec execute 201 sec latency 66.975 ms 2024-09-21T17:28:17.368 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:17 smithi018 ceph-mon[63924]: pgmap v26: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-21T17:28:17.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:17 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:17.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:17 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:17.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:17 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T17:28:17.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:17 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T17:28:17.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:17 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:17.397 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:17 smithi157 ceph-mon[49460]: pgmap v26: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-21T17:28:17.397 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:17 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:17.397 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:17 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:17.397 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:17 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi157.oowooq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T17:28:17.397 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:17 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T17:28:17.397 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:17 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:17.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 129417 9.13 MB/sec execute 202 sec latency 68.618 ms 2024-09-21T17:28:18.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:18 smithi157 ceph-mon[49460]: Reconfiguring mgr.smithi157.oowooq (monmap changed)... 2024-09-21T17:28:18.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:18 smithi157 ceph-mon[49460]: Reconfiguring daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:28:18.390 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:18 smithi018 ceph-mon[63924]: Reconfiguring mgr.smithi157.oowooq (monmap changed)... 2024-09-21T17:28:18.390 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:18 smithi018 ceph-mon[63924]: Reconfiguring daemon mgr.smithi157.oowooq on smithi157 2024-09-21T17:28:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 129698 9.10 MB/sec execute 203 sec latency 72.993 ms 2024-09-21T17:28:19.184 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:19 smithi018 ceph-mon[63924]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 591 op/s 2024-09-21T17:28:19.184 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:19.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:19.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T17:28:19.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T17:28:19.185 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:19.256 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:19 smithi157 ceph-mon[49460]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 591 op/s 2024-09-21T17:28:19.256 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:19.256 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:19.256 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T17:28:19.256 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T17:28:19.257 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:19.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 130026 9.11 MB/sec execute 204 sec latency 73.696 ms 2024-09-21T17:28:20.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:20 smithi018 ceph-mon[63924]: Reconfiguring mon.smithi157 (monmap changed)... 2024-09-21T17:28:20.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:20 smithi018 ceph-mon[63924]: Reconfiguring daemon mon.smithi157 on smithi157 2024-09-21T17:28:20.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:20 smithi157 ceph-mon[49460]: Reconfiguring mon.smithi157 (monmap changed)... 2024-09-21T17:28:20.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:20 smithi157 ceph-mon[49460]: Reconfiguring daemon mon.smithi157 on smithi157 2024-09-21T17:28:20.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 130315 9.10 MB/sec execute 205 sec latency 44.730 ms 2024-09-21T17:28:21.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:21 smithi018 ceph-mon[63924]: pgmap v28: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 397 op/s 2024-09-21T17:28:21.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:21.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:21.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:21 smithi018 ceph-mon[63924]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T17:28:21.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T17:28:21.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:21.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:21 smithi018 ceph-mon[63924]: Reconfiguring daemon osd.4 on smithi157 2024-09-21T17:28:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:21 smithi157 ceph-mon[49460]: pgmap v28: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 397 op/s 2024-09-21T17:28:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:21 smithi157 ceph-mon[49460]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T17:28:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T17:28:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:21 smithi157 ceph-mon[49460]: Reconfiguring daemon osd.4 on smithi157 2024-09-21T17:28:21.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 130615 9.13 MB/sec execute 206 sec latency 76.423 ms 2024-09-21T17:28:22.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 130953 9.13 MB/sec execute 207 sec latency 52.479 ms 2024-09-21T17:28:23.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:23 smithi018 ceph-mon[63924]: pgmap v29: 97 pgs: 97 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.0 MiB/s wr, 587 op/s 2024-09-21T17:28:23.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:23.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:23.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T17:28:23.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:23.424 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:28:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:23 smithi157 ceph-mon[49460]: pgmap v29: 97 pgs: 97 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.0 MiB/s wr, 587 op/s 2024-09-21T17:28:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T17:28:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:23.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 131275 9.12 MB/sec execute 208 sec latency 51.955 ms 2024-09-21T17:28:24.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:24 smithi157 ceph-mon[49460]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T17:28:24.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:24 smithi157 ceph-mon[49460]: Reconfiguring daemon osd.5 on smithi157 2024-09-21T17:28:24.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:24.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:24.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T17:28:24.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:24.362 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:24 smithi018 ceph-mon[63924]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T17:28:24.362 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:24 smithi018 ceph-mon[63924]: Reconfiguring daemon osd.5 on smithi157 2024-09-21T17:28:24.362 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:24.362 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:24.362 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T17:28:24.363 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (3m) 45s ago 12m 20.0M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (12m) 45s ago 12m 6405k - 17.2.0 e1d6a67b021e 1a85419d51bd 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (11m) 32s ago 11m 6409k - 17.2.0 e1d6a67b021e 483aac88b5ba 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (2m) 45s ago 12m 46.3M - dad864ee21e9 48d78bd31599 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (6m) 45s ago 6m 37.3M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (6m) 32s ago 6m 15.5M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (2m) 45s ago 13m 489M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:28:24.444 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (81s) 32s ago 11m 455M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (63s) 45s ago 13m 30.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (37s) 32s ago 11m 24.1M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (5m) 45s ago 5m 261M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (5m) 32s ago 5m 86.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (2m) 45s ago 11m 9.99M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (2m) 32s ago 11m 11.7M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (10m) 45s ago 10m 446M 1937M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (10m) 45s ago 10m 352M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (9m) 45s ago 9m 294M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (9m) 45s ago 9m 401M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (8m) 32s ago 8m 416M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:28:24.445 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (8m) 32s ago 8m 451M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:28:24.446 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (8m) 32s ago 8m 405M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:28:24.446 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (7m) 32s ago 7m 515M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:28:24.446 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (88s) 45s ago 11m 46.4M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:28:24.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 131534 9.10 MB/sec execute 209 sec latency 73.334 ms 2024-09-21T17:28:25.164 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:28:25.164 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:28:25.164 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:28:25.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:28:25.164 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:28:25.164 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:28:25.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 4 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:28:25.165 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:28:25.389 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:25 smithi018 ceph-mon[63924]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:25.389 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:25 smithi018 ceph-mon[63924]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:25.389 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:25 smithi018 ceph-mon[63924]: pgmap v30: 97 pgs: 97 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-09-21T17:28:25.390 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:25 smithi018 ceph-mon[63924]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T17:28:25.390 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:25 smithi018 ceph-mon[63924]: Reconfiguring daemon osd.6 on smithi157 2024-09-21T17:28:25.390 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:25 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/47730378' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:28:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:25 smithi157 ceph-mon[49460]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:25 smithi157 ceph-mon[49460]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:25 smithi157 ceph-mon[49460]: pgmap v30: 97 pgs: 97 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-09-21T17:28:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:25 smithi157 ceph-mon[49460]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T17:28:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:25 smithi157 ceph-mon[49460]: Reconfiguring daemon osd.6 on smithi157 2024-09-21T17:28:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:25 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/47730378' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:28:25.614 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:28:25.614 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:28:25.614 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:28:25.614 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:28:25.614 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:28:25.614 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:28:25.614 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:28:25.614 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:28:25.615 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "4/23 daemons upgraded", 2024-09-21T17:28:25.615 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T17:28:25.615 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:28:25.615 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:28:25.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 131789 9.08 MB/sec execute 210 sec latency 63.109 ms 2024-09-21T17:28:26.130 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:28:26.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:26 smithi018 ceph-mon[63924]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:26.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:26 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:26.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:26 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:26.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:26 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T17:28:26.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:26 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:26.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:26 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3045623218' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:28:26.479 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:26 smithi157 ceph-mon[49460]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:26.479 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:26 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:26.479 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:26 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:26.479 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:26 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T17:28:26.479 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:26 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:26.479 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:26 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3045623218' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:28:26.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 132054 9.07 MB/sec execute 211 sec latency 103.667 ms 2024-09-21T17:28:27.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:27 smithi018 ceph-mon[63924]: from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:27.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:27 smithi018 ceph-mon[63924]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T17:28:27.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:27 smithi018 ceph-mon[63924]: Reconfiguring daemon osd.7 on smithi157 2024-09-21T17:28:27.443 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:27 smithi018 ceph-mon[63924]: pgmap v31: 97 pgs: 97 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.2 MiB/s wr, 469 op/s 2024-09-21T17:28:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:27 smithi157 ceph-mon[49460]: from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:27 smithi157 ceph-mon[49460]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T17:28:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:27 smithi157 ceph-mon[49460]: Reconfiguring daemon osd.7 on smithi157 2024-09-21T17:28:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:27 smithi157 ceph-mon[49460]: pgmap v31: 97 pgs: 97 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.2 MiB/s wr, 469 op/s 2024-09-21T17:28:27.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 132353 9.06 MB/sec execute 212 sec latency 38.530 ms 2024-09-21T17:28:28.747 INFO:teuthology.orchestra.run.smithi018.stdout: 5 132580 9.05 MB/sec execute 213 sec latency 74.963 ms 2024-09-21T17:28:29.026 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:28 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:29.026 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:28 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:29.026 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:28 smithi018 ceph-mon[63924]: Reconfiguring mds.foofs.smithi157.iijaxo (monmap changed)... 2024-09-21T17:28:29.026 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:28 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi157.iijaxo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T17:28:29.026 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:28 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:29.026 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:28 smithi018 ceph-mon[63924]: Reconfiguring daemon mds.foofs.smithi157.iijaxo on smithi157 2024-09-21T17:28:29.027 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:28 smithi018 ceph-mon[63924]: pgmap v32: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.3 MiB/s wr, 547 op/s 2024-09-21T17:28:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:28 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:28 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:28 smithi157 ceph-mon[49460]: Reconfiguring mds.foofs.smithi157.iijaxo (monmap changed)... 2024-09-21T17:28:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:28 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi157.iijaxo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T17:28:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:28 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:28 smithi157 ceph-mon[49460]: Reconfiguring daemon mds.foofs.smithi157.iijaxo on smithi157 2024-09-21T17:28:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:28 smithi157 ceph-mon[49460]: pgmap v32: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.3 MiB/s wr, 547 op/s 2024-09-21T17:28:29.748 INFO:teuthology.orchestra.run.smithi018.stdout: 5 132932 9.06 MB/sec execute 214 sec latency 58.622 ms 2024-09-21T17:28:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: 5 133366 9.05 MB/sec execute 215 sec latency 42.932 ms 2024-09-21T17:28:30.812 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:30.812 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:30.812 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: pgmap v33: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.7 MiB/s wr, 366 op/s 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi018"} : dispatch 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi018"}]': finished 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi157"} : dispatch 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi157"}]': finished 2024-09-21T17:28:30.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:28:31.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:31.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:31.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: pgmap v33: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.7 MiB/s wr, 366 op/s 2024-09-21T17:28:31.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:28:31.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:28:31.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:28:31.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:31.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi018"} : dispatch 2024-09-21T17:28:31.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi018"}]': finished 2024-09-21T17:28:31.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi157"} : dispatch 2024-09-21T17:28:31.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi157"}]': finished 2024-09-21T17:28:31.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:28:31.748 INFO:teuthology.orchestra.run.smithi018.stdout: 5 133701 9.06 MB/sec execute 216 sec latency 43.433 ms 2024-09-21T17:28:31.822 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:31 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all mon 2024-09-21T17:28:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:31 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all mon 2024-09-21T17:28:32.748 INFO:teuthology.orchestra.run.smithi018.stdout: 5 134014 9.06 MB/sec execute 217 sec latency 54.037 ms 2024-09-21T17:28:32.921 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:32 smithi018 ceph-mon[63924]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 560 op/s 2024-09-21T17:28:32.921 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:32 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:32.921 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:32 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi018", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T17:28:32.921 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:32 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:32 smithi157 ceph-mon[49460]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 560 op/s 2024-09-21T17:28:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:32 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:32 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi018", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T17:28:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:32 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:33.748 INFO:teuthology.orchestra.run.smithi018.stdout: 5 134335 9.08 MB/sec execute 218 sec latency 54.036 ms 2024-09-21T17:28:33.934 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:33 smithi018 ceph-mon[63924]: Upgrade: Updating crash.smithi018 (1/2) 2024-09-21T17:28:33.934 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:33 smithi018 ceph-mon[63924]: Deploying daemon crash.smithi018 on smithi018 2024-09-21T17:28:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:33 smithi157 ceph-mon[49460]: Upgrade: Updating crash.smithi018 (1/2) 2024-09-21T17:28:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:33 smithi157 ceph-mon[49460]: Deploying daemon crash.smithi018 on smithi018 2024-09-21T17:28:34.748 INFO:teuthology.orchestra.run.smithi018.stdout: 5 134661 9.08 MB/sec execute 219 sec latency 62.057 ms 2024-09-21T17:28:34.945 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:34 smithi018 ceph-mon[63924]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 370 op/s 2024-09-21T17:28:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:34 smithi157 ceph-mon[49460]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 370 op/s 2024-09-21T17:28:35.748 INFO:teuthology.orchestra.run.smithi018.stdout: 5 134925 9.07 MB/sec execute 220 sec latency 86.087 ms 2024-09-21T17:28:36.753 INFO:teuthology.orchestra.run.smithi018.stdout: 5 135143 9.04 MB/sec execute 221 sec latency 80.976 ms 2024-09-21T17:28:37.231 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:37 smithi018 ceph-mon[63924]: pgmap v36: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-09-21T17:28:37.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:37 smithi157 ceph-mon[49460]: pgmap v36: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-09-21T17:28:37.753 INFO:teuthology.orchestra.run.smithi018.stdout: 5 135415 9.03 MB/sec execute 222 sec latency 54.248 ms 2024-09-21T17:28:38.753 INFO:teuthology.orchestra.run.smithi018.stdout: 5 135783 9.03 MB/sec execute 223 sec latency 51.324 ms 2024-09-21T17:28:39.036 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:38 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:39.036 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:38 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:39.036 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:38 smithi018 ceph-mon[63924]: pgmap v37: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 472 op/s 2024-09-21T17:28:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:38 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:38 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:38 smithi157 ceph-mon[49460]: pgmap v37: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 472 op/s 2024-09-21T17:28:39.753 INFO:teuthology.orchestra.run.smithi018.stdout: 5 136114 9.02 MB/sec execute 224 sec latency 46.675 ms 2024-09-21T17:28:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:40 smithi157 ceph-mon[49460]: Upgrade: Updating crash.smithi157 (2/2) 2024-09-21T17:28:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:40 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:40 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi157", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T17:28:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:40 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:40 smithi157 ceph-mon[49460]: Deploying daemon crash.smithi157 on smithi157 2024-09-21T17:28:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:40 smithi157 ceph-mon[49460]: pgmap v38: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-21T17:28:40.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:40 smithi018 ceph-mon[63924]: Upgrade: Updating crash.smithi157 (2/2) 2024-09-21T17:28:40.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:40.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi157", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T17:28:40.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:40 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:28:40.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:40 smithi018 ceph-mon[63924]: Deploying daemon crash.smithi157 on smithi157 2024-09-21T17:28:40.554 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:40 smithi018 ceph-mon[63924]: pgmap v38: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-21T17:28:40.753 INFO:teuthology.orchestra.run.smithi018.stdout: 5 136638 9.06 MB/sec execute 225 sec latency 43.901 ms 2024-09-21T17:28:41.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 137151 9.06 MB/sec execute 226 sec latency 32.353 ms 2024-09-21T17:28:42.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 137573 9.09 MB/sec execute 227 sec latency 56.920 ms 2024-09-21T17:28:43.100 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:43 smithi018 ceph-mon[63924]: pgmap v39: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 597 op/s 2024-09-21T17:28:43.339 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:43 smithi157 ceph-mon[49460]: pgmap v39: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 597 op/s 2024-09-21T17:28:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 137965 9.12 MB/sec execute 228 sec latency 41.969 ms 2024-09-21T17:28:44.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 138325 9.12 MB/sec execute 229 sec latency 121.596 ms 2024-09-21T17:28:44.877 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:44.877 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:44.877 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:28:44.878 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:44 smithi018 ceph-mon[63924]: pgmap v40: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-21T17:28:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:44 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:44 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:44 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:28:45.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:44 smithi157 ceph-mon[49460]: pgmap v40: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-21T17:28:45.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 138654 9.10 MB/sec execute 230 sec latency 42.029 ms 2024-09-21T17:28:45.882 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:45 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:28:46.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:45 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:28:46.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 139098 9.11 MB/sec execute 231 sec latency 39.597 ms 2024-09-21T17:28:46.896 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:46 smithi018 ceph-mon[63924]: pgmap v41: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 489 op/s 2024-09-21T17:28:47.223 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:46 smithi157 ceph-mon[49460]: pgmap v41: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 489 op/s 2024-09-21T17:28:47.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 139586 9.12 MB/sec execute 232 sec latency 54.062 ms 2024-09-21T17:28:48.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 139980 9.13 MB/sec execute 233 sec latency 75.585 ms 2024-09-21T17:28:49.174 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:49 smithi018 ceph-mon[63924]: pgmap v42: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.7 MiB/s wr, 642 op/s 2024-09-21T17:28:49.174 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:49 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:49.174 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:49 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:49.175 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:49 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:49.473 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:49 smithi157 ceph-mon[49460]: pgmap v42: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.7 MiB/s wr, 642 op/s 2024-09-21T17:28:49.473 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:49 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:49.473 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:49 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:49.473 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:49 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:49.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 140381 9.13 MB/sec execute 234 sec latency 72.061 ms 2024-09-21T17:28:50.186 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:50 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:50.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:50 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:50.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 140740 9.13 MB/sec execute 235 sec latency 53.233 ms 2024-09-21T17:28:51.100 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:51 smithi157 ceph-mon[49460]: pgmap v43: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 460 op/s 2024-09-21T17:28:51.237 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:51 smithi018 ceph-mon[63924]: pgmap v43: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 460 op/s 2024-09-21T17:28:51.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 141151 9.16 MB/sec execute 236 sec latency 52.267 ms 2024-09-21T17:28:52.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 141431 9.16 MB/sec execute 237 sec latency 86.438 ms 2024-09-21T17:28:53.276 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:53 smithi018 ceph-mon[63924]: pgmap v44: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 709 op/s 2024-09-21T17:28:53.276 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:53.276 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:53.276 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:53.276 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:53 smithi157 ceph-mon[49460]: pgmap v44: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 709 op/s 2024-09-21T17:28:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:53.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 141726 9.17 MB/sec execute 238 sec latency 63.430 ms 2024-09-21T17:28:54.308 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:54.308 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:54.430 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:54.430 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:28:54.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 142035 9.15 MB/sec execute 239 sec latency 56.231 ms 2024-09-21T17:28:55.327 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:55 smithi018 ceph-mon[63924]: pgmap v45: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 489 op/s 2024-09-21T17:28:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:55 smithi157 ceph-mon[49460]: pgmap v45: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 489 op/s 2024-09-21T17:28:55.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 142291 9.13 MB/sec execute 240 sec latency 42.150 ms 2024-09-21T17:28:56.742 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:28:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 142667 9.13 MB/sec execute 241 sec latency 41.869 ms 2024-09-21T17:28:57.367 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:57 smithi018 ceph-mon[63924]: pgmap v46: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 572 op/s 2024-09-21T17:28:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:57 smithi157 ceph-mon[49460]: pgmap v46: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 572 op/s 2024-09-21T17:28:57.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 143059 9.13 MB/sec execute 242 sec latency 38.114 ms 2024-09-21T17:28:57.860 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:28:57.860 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (3m) 5s ago 12m 19.3M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:28:57.860 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (20s) 5s ago 12m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:28:57.860 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (14s) 9s ago 12m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (3m) 5s ago 12m 46.7M - dad864ee21e9 48d78bd31599 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (6m) 5s ago 6m 37.6M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (6m) 9s ago 6m 15.7M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (2m) 5s ago 14m 509M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (115s) 9s ago 12m 455M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (96s) 5s ago 14m 42.4M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (71s) 9s ago 12m 35.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (6m) 5s ago 6m 263M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (6m) 9s ago 6m 86.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (3m) 5s ago 12m 9835k - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (2m) 9s ago 11m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (11m) 5s ago 11m 494M 1937M 17.2.0 e1d6a67b021e f0d119197999 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (10m) 5s ago 10m 423M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (10m) 5s ago 10m 330M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:28:57.861 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (9m) 5s ago 9m 459M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:28:57.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (9m) 9s ago 9m 483M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:28:57.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (9m) 9s ago 9m 536M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:28:57.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (8m) 9s ago 8m 456M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:28:57.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (8m) 9s ago 8m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:28:57.862 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (2m) 5s ago 11m 47.2M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:28:58.371 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:58 smithi018 ceph-mon[63924]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:58.492 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:28:58.492 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:28:58.492 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:28:58.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:28:58.492 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:28:58.492 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:28:58.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 4 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:28:58.493 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:28:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:58 smithi157 ceph-mon[49460]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:58.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 143427 9.14 MB/sec execute 243 sec latency 39.936 ms 2024-09-21T17:28:59.005 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:28:59.005 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:28:59.005 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:28:59.005 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:28:59.005 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:28:59.006 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:28:59.006 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:28:59.006 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:28:59.006 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:28:59.006 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "6/23 daemons upgraded", 2024-09-21T17:28:59.006 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading crash daemons", 2024-09-21T17:28:59.006 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:28:59.006 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:28:59.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:59 smithi018 ceph-mon[63924]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:59.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:59 smithi018 ceph-mon[63924]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:59.130 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:59 smithi018 ceph-mon[63924]: pgmap v47: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 598 op/s 2024-09-21T17:28:59.131 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:28:59 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2704736929' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:28:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:59 smithi157 ceph-mon[49460]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:59 smithi157 ceph-mon[49460]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:28:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:59 smithi157 ceph-mon[49460]: pgmap v47: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 598 op/s 2024-09-21T17:28:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:28:59 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2704736929' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:28:59.653 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:28:59.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 143791 9.13 MB/sec execute 244 sec latency 61.841 ms 2024-09-21T17:29:00.135 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:00 smithi018 ceph-mon[63924]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:00.135 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:00.135 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:00.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:29:00.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:29:00.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:00.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:00.136 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:00 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1026895489' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:29:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:00 smithi157 ceph-mon[49460]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:29:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:29:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:00 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1026895489' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:29:00.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 144210 9.14 MB/sec execute 245 sec latency 70.885 ms 2024-09-21T17:29:01.172 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: pgmap v48: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-21T17:29:01.172 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:29:01.172 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all crash 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi018"} : dispatch 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi018"}]': finished 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi157"} : dispatch 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi157"}]': finished 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T17:29:01.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: Upgrade: osd.0 is safe to restart 2024-09-21T17:29:01.174 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:29:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: pgmap v48: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-21T17:29:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:29:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all crash 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi018"} : dispatch 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi018"}]': finished 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi157"} : dispatch 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi157"}]': finished 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: Upgrade: osd.0 is safe to restart 2024-09-21T17:29:01.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:29:01.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 144420 9.13 MB/sec execute 246 sec latency 85.174 ms 2024-09-21T17:29:02.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 144734 9.15 MB/sec execute 247 sec latency 44.225 ms 2024-09-21T17:29:03.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:03 smithi018 ceph-mon[63924]: pgmap v49: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 664 op/s 2024-09-21T17:29:03.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:03 smithi018 ceph-mon[63924]: Upgrade: Updating osd.0 2024-09-21T17:29:03.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:03.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T17:29:03.214 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:29:03.214 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:03 smithi018 ceph-mon[63924]: Deploying daemon osd.0 on smithi018 2024-09-21T17:29:03.214 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:03 smithi018 ceph-mon[63924]: mgrmap e43: smithi018.jndyyn(active, since 92s), standbys: smithi157.oowooq 2024-09-21T17:29:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:03 smithi157 ceph-mon[49460]: pgmap v49: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 664 op/s 2024-09-21T17:29:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:03 smithi157 ceph-mon[49460]: Upgrade: Updating osd.0 2024-09-21T17:29:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T17:29:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:29:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:03 smithi157 ceph-mon[49460]: Deploying daemon osd.0 on smithi018 2024-09-21T17:29:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:03 smithi157 ceph-mon[49460]: mgrmap e43: smithi018.jndyyn(active, since 92s), standbys: smithi157.oowooq 2024-09-21T17:29:03.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 144977 9.14 MB/sec execute 248 sec latency 46.128 ms 2024-09-21T17:29:04.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 145258 9.15 MB/sec execute 249 sec latency 77.080 ms 2024-09-21T17:29:05.149 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:05 smithi018 ceph-mon[63924]: pgmap v50: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 415 op/s 2024-09-21T17:29:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:05 smithi157 ceph-mon[49460]: pgmap v50: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 415 op/s 2024-09-21T17:29:05.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 145633 9.13 MB/sec execute 250 sec latency 35.628 ms 2024-09-21T17:29:06.418 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:06 smithi018 systemd[1]: Stopping Ceph osd.0 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:29:06.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 145903 9.11 MB/sec execute 251 sec latency 43.428 ms 2024-09-21T17:29:06.920 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:06 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[33928]: 2024-09-21T17:29:06.711+0000 7f6eae283700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:29:06.920 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:06 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[33928]: 2024-09-21T17:29:06.711+0000 7f6eae283700 -1 osd.0 74 *** Got signal Terminated *** 2024-09-21T17:29:06.921 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:06 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[33928]: 2024-09-21T17:29:06.711+0000 7f6eae283700 -1 osd.0 74 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:29:07.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:07 smithi018 ceph-mon[63924]: pgmap v51: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 505 op/s 2024-09-21T17:29:07.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:07 smithi018 ceph-mon[63924]: osd.0 marked itself down and dead 2024-09-21T17:29:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:07 smithi157 ceph-mon[49460]: pgmap v51: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 505 op/s 2024-09-21T17:29:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:07 smithi157 ceph-mon[49460]: osd.0 marked itself down and dead 2024-09-21T17:29:07.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 146048 9.09 MB/sec execute 252 sec latency 673.314 ms 2024-09-21T17:29:07.999 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:07 smithi018 podman[68259]: 2024-09-21 17:29:07.728567209 +0000 UTC m=+1.156474569 container died f0d1191979998d3cc9ee2136e59c88c9c35887b9892e1bc99b9330eaaba42f01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.openshift.tags=base centos centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, ceph=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., architecture=x86_64) 2024-09-21T17:29:08.258 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:08 smithi018 ceph-mon[63924]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:29:08.258 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:08 smithi018 ceph-mon[63924]: osdmap e75: 8 total, 7 up, 8 in 2024-09-21T17:29:08.258 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:08 smithi018 podman[68259]: 2024-09-21 17:29:08.121871977 +0000 UTC m=+1.549779329 container cleanup f0d1191979998d3cc9ee2136e59c88c9c35887b9892e1bc99b9330eaaba42f01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, version=8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.buildah.version=1.19.8, vcs-type=git, architecture=x86_64, release=754, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc.) 2024-09-21T17:29:08.259 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:08 smithi018 bash[68259]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0 2024-09-21T17:29:08.512 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:08 smithi018 podman[68276]: 2024-09-21 17:29:08.490913308 +0000 UTC m=+0.754278420 container remove f0d1191979998d3cc9ee2136e59c88c9c35887b9892e1bc99b9330eaaba42f01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, name=centos-stream, GIT_CLEAN=True, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T17:29:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:08 smithi157 ceph-mon[49460]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:29:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:08 smithi157 ceph-mon[49460]: osdmap e75: 8 total, 7 up, 8 in 2024-09-21T17:29:08.754 INFO:teuthology.orchestra.run.smithi018.stdout: 5 146358 9.09 MB/sec execute 253 sec latency 1067.943 ms 2024-09-21T17:29:09.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:09 smithi018 ceph-mon[63924]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 480 op/s 2024-09-21T17:29:09.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:09 smithi018 ceph-mon[63924]: osdmap e76: 8 total, 7 up, 8 in 2024-09-21T17:29:09.270 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:09 smithi018 podman[68363]: 2024-09-21 17:29:08.961790782 +0000 UTC m=+0.024439479 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:29:09.270 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:09 smithi018 podman[68363]: 2024-09-21 17:29:09.184084747 +0000 UTC m=+0.246733432 container create 375471712e51dd570fd21b1f3588bf31ddf2a4225880d89f30f8e95bf96c38d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:29:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:09 smithi157 ceph-mon[49460]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 480 op/s 2024-09-21T17:29:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:09 smithi157 ceph-mon[49460]: osdmap e76: 8 total, 7 up, 8 in 2024-09-21T17:29:09.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 146763 9.08 MB/sec execute 254 sec latency 59.758 ms 2024-09-21T17:29:09.772 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:09 smithi018 podman[68363]: 2024-09-21 17:29:09.523283906 +0000 UTC m=+0.585932604 container init 375471712e51dd570fd21b1f3588bf31ddf2a4225880d89f30f8e95bf96c38d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:29:09.772 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:09 smithi018 podman[68363]: 2024-09-21 17:29:09.532478407 +0000 UTC m=+0.595127096 container start 375471712e51dd570fd21b1f3588bf31ddf2a4225880d89f30f8e95bf96c38d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T17:29:09.773 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:09 smithi018 podman[68363]: 2024-09-21 17:29:09.707090197 +0000 UTC m=+0.769738899 container attach 375471712e51dd570fd21b1f3588bf31ddf2a4225880d89f30f8e95bf96c38d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T17:29:10.023 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:09 smithi018 podman[68363]: 2024-09-21 17:29:09.872575081 +0000 UTC m=+0.935223773 container died 375471712e51dd570fd21b1f3588bf31ddf2a4225880d89f30f8e95bf96c38d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:29:10.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 147196 9.10 MB/sec execute 255 sec latency 46.333 ms 2024-09-21T17:29:11.039 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:10 smithi018 podman[68384]: 2024-09-21 17:29:10.806103595 +0000 UTC m=+0.917131658 container remove 375471712e51dd570fd21b1f3588bf31ddf2a4225880d89f30f8e95bf96c38d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T17:29:11.039 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:10 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T17:29:11.040 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:10 smithi018 systemd[1]: Stopped Ceph osd.0 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:29:11.040 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:10 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.0.service: Consumed 2min 17.735s CPU time. 2024-09-21T17:29:11.291 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:11 smithi018 ceph-mon[63924]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.8 MiB/s wr, 269 op/s 2024-09-21T17:29:11.292 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:11 smithi018 systemd[1]: Starting Ceph osd.0 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:29:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:11 smithi157 ceph-mon[49460]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.8 MiB/s wr, 269 op/s 2024-09-21T17:29:11.544 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:11 smithi018 podman[68480]: 2024-09-21 17:29:11.352962519 +0000 UTC m=+0.024796436 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:29:11.544 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:11 smithi018 podman[68480]: 2024-09-21 17:29:11.482133767 +0000 UTC m=+0.153967659 container create 05add7e531fc971e2bc238bad23cd80bc191605ce192baf80eb3c34b27235865 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T17:29:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 147578 9.09 MB/sec execute 256 sec latency 51.551 ms 2024-09-21T17:29:12.056 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:11 smithi018 podman[68480]: 2024-09-21 17:29:11.826838488 +0000 UTC m=+0.498672381 container init 05add7e531fc971e2bc238bad23cd80bc191605ce192baf80eb3c34b27235865 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:29:12.056 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:11 smithi018 podman[68480]: 2024-09-21 17:29:11.83760323 +0000 UTC m=+0.509437124 container start 05add7e531fc971e2bc238bad23cd80bc191605ce192baf80eb3c34b27235865 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:29:12.056 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:12 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:29:12.057 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:12 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:29:12.307 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:12 smithi018 ceph-mon[63924]: Health check failed: Degraded data redundancy: 216/1671 objects degraded (12.926%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:12.307 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:12 smithi018 bash[68480]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:29:12.307 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:12 smithi018 bash[68480]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:29:12.307 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:12 smithi018 podman[68480]: 2024-09-21 17:29:12.118706006 +0000 UTC m=+0.790539911 container attach 05add7e531fc971e2bc238bad23cd80bc191605ce192baf80eb3c34b27235865 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T17:29:12.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:12 smithi157 ceph-mon[49460]: Health check failed: Degraded data redundancy: 216/1671 objects degraded (12.926%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:12.758 INFO:teuthology.orchestra.run.smithi018.stdout: 5 147895 9.11 MB/sec execute 257 sec latency 80.177 ms 2024-09-21T17:29:13.370 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:13 smithi018 ceph-mon[63924]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 566 op/s; 216/1671 objects degraded (12.926%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:13 smithi157 ceph-mon[49460]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 566 op/s; 216/1671 objects degraded (12.926%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:13.619 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T17:29:13.620 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 bash[68480]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T17:29:13.620 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T17:29:13.620 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 bash[68480]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T17:29:13.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 148222 9.12 MB/sec execute 258 sec latency 48.523 ms 2024-09-21T17:29:13.872 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T17:29:13.872 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 bash[68480]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T17:29:13.872 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T17:29:13.872 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 bash[68480]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T17:29:13.872 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T17:29:13.872 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 bash[68480]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T17:29:13.872 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T17:29:13.872 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 bash[68480]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T17:29:14.124 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate[68494]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T17:29:14.124 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 bash[68480]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T17:29:14.124 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:13 smithi018 podman[68480]: 2024-09-21 17:29:13.906562462 +0000 UTC m=+2.578396375 container died 05add7e531fc971e2bc238bad23cd80bc191605ce192baf80eb3c34b27235865 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:29:14.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 148502 9.12 MB/sec execute 259 sec latency 53.379 ms 2024-09-21T17:29:14.900 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:14 smithi018 podman[68645]: 2024-09-21 17:29:14.684668908 +0000 UTC m=+0.770570238 container remove 05add7e531fc971e2bc238bad23cd80bc191605ce192baf80eb3c34b27235865 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-activate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T17:29:15.162 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:14 smithi018 podman[68704]: 2024-09-21 17:29:14.825048062 +0000 UTC m=+0.019969369 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:29:15.162 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:14 smithi018 podman[68704]: 2024-09-21 17:29:14.983843721 +0000 UTC m=+0.178765018 container create ac5dc125292239739ae47b627670fefc6c6c9c67495a40870fd918e4e0f1a458 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T17:29:15.413 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:15 smithi018 ceph-mon[63924]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 431 op/s; 216/1671 objects degraded (12.926%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:15.414 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:15 smithi018 podman[68704]: 2024-09-21 17:29:15.353818498 +0000 UTC m=+0.548739798 container init ac5dc125292239739ae47b627670fefc6c6c9c67495a40870fd918e4e0f1a458 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T17:29:15.414 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:15 smithi018 podman[68704]: 2024-09-21 17:29:15.367190772 +0000 UTC m=+0.562112073 container start ac5dc125292239739ae47b627670fefc6c6c9c67495a40870fd918e4e0f1a458 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2) 2024-09-21T17:29:15.414 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:15 smithi018 ceph-osd[68722]: -- 172.21.15.18:0/2166358455 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5595620d41e0 con 0x5595620d0000 2024-09-21T17:29:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:15 smithi157 ceph-mon[49460]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 431 op/s; 216/1671 objects degraded (12.926%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:15.664 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:15 smithi018 bash[68704]: ac5dc125292239739ae47b627670fefc6c6c9c67495a40870fd918e4e0f1a458 2024-09-21T17:29:15.664 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:15 smithi018 systemd[1]: Started Ceph osd.0 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:29:15.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 148819 9.13 MB/sec execute 260 sec latency 65.366 ms 2024-09-21T17:29:16.692 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:16.692 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:29:16.692 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:16.693 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:16.693 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:29:16.693 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:16 smithi018 ceph-mon[63924]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.6 MiB/s wr, 285 op/s; 210/1665 objects degraded (12.613%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:16.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 149102 9.11 MB/sec execute 261 sec latency 44.377 ms 2024-09-21T17:29:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:29:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:29:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:16 smithi157 ceph-mon[49460]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.6 MiB/s wr, 285 op/s; 210/1665 objects degraded (12.613%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:17.449 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:17 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[68718]: 2024-09-21T17:29:17.195+0000 7f811bc74640 -1 Falling back to public interface 2024-09-21T17:29:17.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 149344 9.09 MB/sec execute 262 sec latency 43.804 ms 2024-09-21T17:29:18.207 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:18 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 221/1695 objects degraded (13.038%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:18.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:18 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 221/1695 objects degraded (13.038%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:18.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 149643 9.08 MB/sec execute 263 sec latency 60.242 ms 2024-09-21T17:29:19.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:19 smithi018 ceph-mon[63924]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 464 op/s; 221/1695 objects degraded (13.038%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:19 smithi157 ceph-mon[49460]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 464 op/s; 221/1695 objects degraded (13.038%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:19.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 150006 9.08 MB/sec execute 264 sec latency 58.980 ms 2024-09-21T17:29:20.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 150306 9.07 MB/sec execute 265 sec latency 53.605 ms 2024-09-21T17:29:21.283 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:21 smithi018 ceph-mon[63924]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 221/1695 objects degraded (13.038%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:21 smithi157 ceph-mon[49460]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 221/1695 objects degraded (13.038%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:21.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 150649 9.08 MB/sec execute 266 sec latency 68.817 ms 2024-09-21T17:29:22.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 151110 9.08 MB/sec execute 267 sec latency 51.416 ms 2024-09-21T17:29:23.332 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:23 smithi018 ceph-mon[63924]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.7 MiB/s wr, 580 op/s; 219/1716 objects degraded (12.762%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:23 smithi157 ceph-mon[49460]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.7 MiB/s wr, 580 op/s; 219/1716 objects degraded (12.762%); 0 B/s, 0 objects/s recovering 2024-09-21T17:29:23.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 151604 9.10 MB/sec execute 268 sec latency 42.327 ms 2024-09-21T17:29:24.344 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:24 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[68718]: 2024-09-21T17:29:24.091+0000 7f811bc74640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-21T17:29:24.755 INFO:teuthology.orchestra.run.smithi018.stdout: 5 152040 9.13 MB/sec execute 269 sec latency 42.723 ms 2024-09-21T17:29:25.117 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:25.117 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:25.117 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:24 smithi018 ceph-mon[63924]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 381 op/s; 219/1716 objects degraded (12.762%) 2024-09-21T17:29:25.117 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:25.118 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:25.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:25.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:25.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:24 smithi157 ceph-mon[49460]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 381 op/s; 219/1716 objects degraded (12.762%) 2024-09-21T17:29:25.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:25.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:25.368 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:25 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[68718]: 2024-09-21T17:29:25.248+0000 7f811bc74640 -1 osd.0 74 log_to_monitors true 2024-09-21T17:29:25.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 152390 9.14 MB/sec execute 270 sec latency 38.259 ms 2024-09-21T17:29:26.121 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:25 smithi018 ceph-mon[63924]: from='osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T17:29:26.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:25 smithi157 ceph-mon[49460]: from='osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T17:29:26.377 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:29:26 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[68718]: 2024-09-21T17:29:26.341+0000 7f810fa16640 -1 osd.0 74 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:29:26.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 152669 9.12 MB/sec execute 271 sec latency 53.150 ms 2024-09-21T17:29:27.144 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:26 smithi018 ceph-mon[63924]: from='osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T17:29:27.144 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:26 smithi018 ceph-mon[63924]: osdmap e77: 8 total, 7 up, 8 in 2024-09-21T17:29:27.144 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:26 smithi018 ceph-mon[63924]: from='osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:29:27.144 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:26 smithi018 ceph-mon[63924]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 499 op/s; 225/1710 objects degraded (13.158%) 2024-09-21T17:29:27.144 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:26 smithi018 ceph-mon[63924]: from='osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902]' entity='osd.0' 2024-09-21T17:29:27.145 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:26 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 219/1716 objects degraded (12.762%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:26 smithi157 ceph-mon[49460]: from='osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T17:29:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:26 smithi157 ceph-mon[49460]: osdmap e77: 8 total, 7 up, 8 in 2024-09-21T17:29:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:26 smithi157 ceph-mon[49460]: from='osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:29:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:26 smithi157 ceph-mon[49460]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 499 op/s; 225/1710 objects degraded (13.158%) 2024-09-21T17:29:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:26 smithi157 ceph-mon[49460]: from='osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902]' entity='osd.0' 2024-09-21T17:29:27.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:26 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 219/1716 objects degraded (12.762%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:27.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 152825 9.10 MB/sec execute 272 sec latency 449.220 ms 2024-09-21T17:29:28.282 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:27 smithi157 ceph-mon[49460]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:29:28.282 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:27 smithi157 ceph-mon[49460]: osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902] boot 2024-09-21T17:29:28.282 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:27 smithi157 ceph-mon[49460]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T17:29:28.282 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T17:29:28.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:27 smithi018 ceph-mon[63924]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:29:28.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:27 smithi018 ceph-mon[63924]: osd.0 [v2:172.21.15.18:6802/1769861902,v1:172.21.15.18:6803/1769861902] boot 2024-09-21T17:29:28.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:27 smithi018 ceph-mon[63924]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T17:29:28.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T17:29:28.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 152825 9.07 MB/sec execute 273 sec latency 1449.253 ms 2024-09-21T17:29:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:29 smithi157 ceph-mon[49460]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 653 op/s; 229/1761 objects degraded (13.004%) 2024-09-21T17:29:29.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:29 smithi157 ceph-mon[49460]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T17:29:29.285 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:29 smithi018 ceph-mon[63924]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 653 op/s; 229/1761 objects degraded (13.004%) 2024-09-21T17:29:29.285 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:29 smithi018 ceph-mon[63924]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T17:29:29.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 152838 9.04 MB/sec execute 274 sec latency 2348.901 ms 2024-09-21T17:29:30.206 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:29:30.376 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T17:29:30.376 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:30.376 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: pgmap v69: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 7.8 MiB/s wr, 725 op/s; 229/1761 objects degraded (13.004%) 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:30.377 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:30 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T17:29:30.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 153118 9.02 MB/sec execute 275 sec latency 58.237 ms 2024-09-21T17:29:30.784 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T17:29:30.784 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:30.784 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:30.784 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:29:30.784 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:29:30.784 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:30.784 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:30.785 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: pgmap v69: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 7.8 MiB/s wr, 725 op/s; 229/1761 objects degraded (13.004%) 2024-09-21T17:29:30.785 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:30.785 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:29:30.785 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:30.785 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:30.785 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:30.785 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:30 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T17:29:31.127 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:29:31.128 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (4m) 7s ago 13m 19.4M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:29:31.128 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (53s) 7s ago 13m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:29:31.128 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (47s) 42s ago 12m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:29:31.128 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (4m) 7s ago 13m 47.0M - dad864ee21e9 48d78bd31599 2024-09-21T17:29:31.128 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (7m) 7s ago 7m 37.6M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:29:31.128 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (7m) 42s ago 7m 15.7M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:29:31.128 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (3m) 7s ago 14m 512M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:29:31.128 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (2m) 42s ago 12m 455M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (2m) 7s ago 14m 45.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (104s) 42s ago 12m 35.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (6m) 7s ago 6m 269M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (6m) 42s ago 6m 86.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (3m) 7s ago 13m 9935k - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (3m) 42s ago 12m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (16s) 7s ago 11m 11.1M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (11m) 7s ago 11m 459M 1937M 17.2.0 e1d6a67b021e bf5b5b4a38cc 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (10m) 7s ago 10m 359M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:29:31.129 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (10m) 7s ago 10m 484M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:29:31.130 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (10m) 42s ago 10m 483M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:29:31.130 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (9m) 42s ago 9m 536M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:29:31.130 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (9m) 42s ago 9m 456M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:29:31.130 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (8m) 42s ago 8m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:29:31.130 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (2m) 7s ago 12m 50.5M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:29:31.647 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:29:31.647 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:29:31.647 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:29:31.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:29:31.648 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:29:31.648 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:29:31.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:29:31.648 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:29:31.648 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T17:29:31.648 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:29:31.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:29:31.649 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:29:31.649 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:29:31.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:29:31.649 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:29:31.649 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-21T17:29:31.649 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 5 2024-09-21T17:29:31.650 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:29:31.650 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:29:31.682 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:31 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T17:29:31.683 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:31 smithi018 ceph-mon[63924]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T17:29:31.683 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:31.683 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:29:31.683 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:31 smithi018 ceph-mon[63924]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:31.683 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:31 smithi018 ceph-mon[63924]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T17:29:31.683 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:31 smithi018 ceph-mon[63924]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:31.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 153332 9.01 MB/sec execute 276 sec latency 1010.291 ms 2024-09-21T17:29:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:31 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T17:29:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:31 smithi157 ceph-mon[49460]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T17:29:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:29:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:31 smithi157 ceph-mon[49460]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:32.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:31 smithi157 ceph-mon[49460]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T17:29:32.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:31 smithi157 ceph-mon[49460]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:32.119 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:29:32.119 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "7/23 daemons upgraded", 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:29:32.120 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:29:32.675 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN Degraded data redundancy: 229/1761 objects degraded (13.004%), 29 pgs degraded 2024-09-21T17:29:32.675 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 229/1761 objects degraded (13.004%), 29 pgs degraded 2024-09-21T17:29:32.675 INFO:teuthology.orchestra.run.smithi018.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-21T17:29:32.675 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-21T17:29:32.675 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-21T17:29:32.675 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-21T17:29:32.675 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.5 is active+undersized+degraded, acting [2,4] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-21T17:29:32.676 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-21T17:29:32.677 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-21T17:29:32.700 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:32 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 229/1761 objects degraded (13.004%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:32.700 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:32 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2495888415' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:32.700 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:32 smithi018 ceph-mon[63924]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T17:29:32.700 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:32 smithi018 ceph-mon[63924]: pgmap v72: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.8 MiB/s wr, 310 op/s; 84/1569 objects degraded (5.354%); 5.1 MiB/s, 44 objects/s recovering 2024-09-21T17:29:32.700 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:32 smithi018 ceph-mon[63924]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:32.758 INFO:teuthology.orchestra.run.smithi018.stdout: 5 153635 9.00 MB/sec execute 277 sec latency 68.236 ms 2024-09-21T17:29:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:32 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 229/1761 objects degraded (13.004%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:32 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2495888415' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:29:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:32 smithi157 ceph-mon[49460]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T17:29:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:32 smithi157 ceph-mon[49460]: pgmap v72: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.8 MiB/s wr, 310 op/s; 84/1569 objects degraded (5.354%); 5.1 MiB/s, 44 objects/s recovering 2024-09-21T17:29:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:32 smithi157 ceph-mon[49460]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:29:33.637 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:33 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3692782338' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:29:33.638 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:33 smithi018 ceph-mon[63924]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T17:29:33.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 153926 9.02 MB/sec execute 278 sec latency 835.815 ms 2024-09-21T17:29:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:33 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3692782338' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:29:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:33 smithi157 ceph-mon[49460]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T17:29:34.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 154201 8.99 MB/sec execute 279 sec latency 921.193 ms 2024-09-21T17:29:34.800 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:34 smithi018 ceph-mon[63924]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T17:29:34.800 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:34 smithi018 ceph-mon[63924]: pgmap v75: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.8 MiB/s wr, 310 op/s; 84/1569 objects degraded (5.354%); 5.1 MiB/s, 44 objects/s recovering 2024-09-21T17:29:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:34 smithi157 ceph-mon[49460]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T17:29:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:34 smithi157 ceph-mon[49460]: pgmap v75: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.8 MiB/s wr, 310 op/s; 84/1569 objects degraded (5.354%); 5.1 MiB/s, 44 objects/s recovering 2024-09-21T17:29:35.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 154530 8.98 MB/sec execute 280 sec latency 271.451 ms 2024-09-21T17:29:35.852 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:35 smithi018 ceph-mon[63924]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T17:29:36.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:35 smithi157 ceph-mon[49460]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T17:29:36.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 154724 8.97 MB/sec execute 281 sec latency 63.856 ms 2024-09-21T17:29:36.860 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:36 smithi018 ceph-mon[63924]: pgmap v77: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 786 KiB/s wr, 68 op/s; 33/1602 objects degraded (2.060%); 0 B/s, 11 objects/s recovering 2024-09-21T17:29:36.860 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:36 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 84/1569 objects degraded (5.354%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:36 smithi157 ceph-mon[49460]: pgmap v77: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 786 KiB/s wr, 68 op/s; 33/1602 objects degraded (2.060%); 0 B/s, 11 objects/s recovering 2024-09-21T17:29:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:36 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 84/1569 objects degraded (5.354%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:37.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 154958 8.96 MB/sec execute 282 sec latency 57.493 ms 2024-09-21T17:29:38.129 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:38 smithi018 ceph-mon[63924]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1602 objects degraded (2.060%), 2 pgs degraded) 2024-09-21T17:29:38.129 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:38 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:29:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:38 smithi157 ceph-mon[49460]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1602 objects degraded (2.060%), 2 pgs degraded) 2024-09-21T17:29:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:38 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:29:38.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 155231 8.98 MB/sec execute 283 sec latency 53.090 ms 2024-09-21T17:29:39.143 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:39 smithi018 ceph-mon[63924]: pgmap v78: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.0 MiB/s wr, 329 op/s; 0 B/s, 16 objects/s recovering 2024-09-21T17:29:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:39 smithi157 ceph-mon[49460]: pgmap v78: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.0 MiB/s wr, 329 op/s; 0 B/s, 16 objects/s recovering 2024-09-21T17:29:39.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 155524 8.98 MB/sec execute 284 sec latency 58.256 ms 2024-09-21T17:29:40.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 155888 9.00 MB/sec execute 285 sec latency 42.343 ms 2024-09-21T17:29:41.215 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:41 smithi018 ceph-mon[63924]: pgmap v79: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.4 MiB/s wr, 271 op/s; 0 B/s, 13 objects/s recovering 2024-09-21T17:29:41.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:41 smithi157 ceph-mon[49460]: pgmap v79: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.4 MiB/s wr, 271 op/s; 0 B/s, 13 objects/s recovering 2024-09-21T17:29:41.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 156159 8.98 MB/sec execute 286 sec latency 62.648 ms 2024-09-21T17:29:42.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 156370 8.96 MB/sec execute 287 sec latency 69.531 ms 2024-09-21T17:29:43.224 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:43 smithi018 ceph-mon[63924]: pgmap v80: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 477 op/s; 0 B/s, 11 objects/s recovering 2024-09-21T17:29:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:43 smithi157 ceph-mon[49460]: pgmap v80: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 477 op/s; 0 B/s, 11 objects/s recovering 2024-09-21T17:29:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 156695 8.95 MB/sec execute 288 sec latency 41.666 ms 2024-09-21T17:29:44.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 157003 8.95 MB/sec execute 289 sec latency 54.079 ms 2024-09-21T17:29:45.258 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:45 smithi018 ceph-mon[63924]: pgmap v81: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.1 MiB/s wr, 394 op/s; 0 B/s, 9 objects/s recovering 2024-09-21T17:29:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:45 smithi157 ceph-mon[49460]: pgmap v81: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.1 MiB/s wr, 394 op/s; 0 B/s, 9 objects/s recovering 2024-09-21T17:29:45.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 157380 8.96 MB/sec execute 290 sec latency 51.020 ms 2024-09-21T17:29:46.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T17:29:46.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:46.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:46.270 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:29:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T17:29:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:29:46.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 157755 8.95 MB/sec execute 291 sec latency 42.784 ms 2024-09-21T17:29:47.276 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:47 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T17:29:47.276 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:47 smithi018 ceph-mon[63924]: Upgrade: osd.1 is safe to restart 2024-09-21T17:29:47.276 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:47 smithi018 ceph-mon[63924]: pgmap v82: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.6 MiB/s wr, 483 op/s; 0 B/s, 8 objects/s recovering 2024-09-21T17:29:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:47 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T17:29:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:47 smithi157 ceph-mon[49460]: Upgrade: osd.1 is safe to restart 2024-09-21T17:29:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:47 smithi157 ceph-mon[49460]: pgmap v82: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.6 MiB/s wr, 483 op/s; 0 B/s, 8 objects/s recovering 2024-09-21T17:29:47.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 158160 8.95 MB/sec execute 292 sec latency 49.518 ms 2024-09-21T17:29:48.305 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:48 smithi018 ceph-mon[63924]: Upgrade: Updating osd.1 2024-09-21T17:29:48.306 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:48.306 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T17:29:48.306 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:29:48.306 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:48 smithi018 ceph-mon[63924]: Deploying daemon osd.1 on smithi018 2024-09-21T17:29:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:48 smithi157 ceph-mon[49460]: Upgrade: Updating osd.1 2024-09-21T17:29:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:29:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T17:29:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:29:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:48 smithi157 ceph-mon[49460]: Deploying daemon osd.1 on smithi018 2024-09-21T17:29:48.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 158415 8.94 MB/sec execute 293 sec latency 43.141 ms 2024-09-21T17:29:49.250 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:49 smithi018 ceph-mon[63924]: pgmap v83: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 496 op/s; 0 B/s, 3 objects/s recovering 2024-09-21T17:29:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:49 smithi157 ceph-mon[49460]: pgmap v83: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 496 op/s; 0 B/s, 3 objects/s recovering 2024-09-21T17:29:49.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 158829 8.98 MB/sec execute 294 sec latency 38.317 ms 2024-09-21T17:29:50.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 159177 8.99 MB/sec execute 295 sec latency 53.209 ms 2024-09-21T17:29:51.361 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:51 smithi018 ceph-mon[63924]: pgmap v84: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 357 op/s 2024-09-21T17:29:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:51 smithi157 ceph-mon[49460]: pgmap v84: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 357 op/s 2024-09-21T17:29:51.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 159489 8.99 MB/sec execute 296 sec latency 57.444 ms 2024-09-21T17:29:52.119 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:52 smithi018 systemd[1]: Stopping Ceph osd.1 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:29:52.623 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:52 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[37035]: 2024-09-21T17:29:52.410+0000 7fb23ae76700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:29:52.623 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:52 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[37035]: 2024-09-21T17:29:52.410+0000 7fb23ae76700 -1 osd.1 85 *** Got signal Terminated *** 2024-09-21T17:29:52.624 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:52 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[37035]: 2024-09-21T17:29:52.410+0000 7fb23ae76700 -1 osd.1 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:29:52.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 159737 8.97 MB/sec execute 297 sec latency 53.391 ms 2024-09-21T17:29:53.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:53 smithi018 ceph-mon[63924]: pgmap v85: 97 pgs: 97 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 563 op/s 2024-09-21T17:29:53.395 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:53 smithi018 ceph-mon[63924]: osd.1 marked itself down and dead 2024-09-21T17:29:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:53 smithi157 ceph-mon[49460]: pgmap v85: 97 pgs: 97 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 563 op/s 2024-09-21T17:29:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:53 smithi157 ceph-mon[49460]: osd.1 marked itself down and dead 2024-09-21T17:29:53.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 159955 8.95 MB/sec execute 298 sec latency 471.969 ms 2024-09-21T17:29:54.260 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:53 smithi018 podman[70615]: 2024-09-21 17:29:53.964792582 +0000 UTC m=+1.694251738 container died bf5b5b4a38cc9d539f590d04c53102949935ab5eb8c0cda71e1507201f4036dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-type=git, release=754, name=centos-stream, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-21T17:29:54.518 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:54 smithi018 ceph-mon[63924]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:29:54.518 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:54 smithi018 ceph-mon[63924]: osdmap e86: 8 total, 7 up, 8 in 2024-09-21T17:29:54.518 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:54 smithi018 ceph-mon[63924]: osdmap e87: 8 total, 7 up, 8 in 2024-09-21T17:29:54.518 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:54 smithi018 podman[70615]: 2024-09-21 17:29:54.409948332 +0000 UTC m=+2.139407478 container cleanup bf5b5b4a38cc9d539f590d04c53102949935ab5eb8c0cda71e1507201f4036dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_CLEAN=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, version=8, ceph=True, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64) 2024-09-21T17:29:54.519 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:54 smithi018 bash[70615]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1 2024-09-21T17:29:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:54 smithi157 ceph-mon[49460]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:29:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:54 smithi157 ceph-mon[49460]: osdmap e86: 8 total, 7 up, 8 in 2024-09-21T17:29:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:54 smithi157 ceph-mon[49460]: osdmap e87: 8 total, 7 up, 8 in 2024-09-21T17:29:54.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 160117 8.92 MB/sec execute 299 sec latency 993.785 ms 2024-09-21T17:29:54.790 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:54 smithi018 podman[70633]: 2024-09-21 17:29:54.673093519 +0000 UTC m=+0.709246067 container remove bf5b5b4a38cc9d539f590d04c53102949935ab5eb8c0cda71e1507201f4036dc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, RELEASE=HEAD, io.buildah.version=1.19.8, ceph=True, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, name=centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_CLEAN=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T17:29:55.300 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:55 smithi018 ceph-mon[63924]: pgmap v87: 97 pgs: 13 stale+active+clean, 84 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 478 op/s 2024-09-21T17:29:55.300 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:55 smithi018 podman[70719]: 2024-09-21 17:29:55.099015435 +0000 UTC m=+0.027732079 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:29:55.300 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:55 smithi018 podman[70719]: 2024-09-21 17:29:55.221926732 +0000 UTC m=+0.150643367 container create 69f8a45e022d739898b7bc4fe679d43bd242694f469be6ad61ce302efa4cbf6f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T17:29:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:55 smithi157 ceph-mon[49460]: pgmap v87: 97 pgs: 13 stale+active+clean, 84 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 478 op/s 2024-09-21T17:29:55.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 160368 8.92 MB/sec execute 300 sec latency 51.174 ms 2024-09-21T17:29:55.837 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:55 smithi018 podman[70719]: 2024-09-21 17:29:55.717476959 +0000 UTC m=+0.646193606 container init 69f8a45e022d739898b7bc4fe679d43bd242694f469be6ad61ce302efa4cbf6f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731) 2024-09-21T17:29:55.837 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:55 smithi018 podman[70719]: 2024-09-21 17:29:55.726498965 +0000 UTC m=+0.655215605 container start 69f8a45e022d739898b7bc4fe679d43bd242694f469be6ad61ce302efa4cbf6f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T17:29:56.102 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:55 smithi018 podman[70719]: 2024-09-21 17:29:55.909239732 +0000 UTC m=+0.837956368 container attach 69f8a45e022d739898b7bc4fe679d43bd242694f469be6ad61ce302efa4cbf6f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-21T17:29:56.103 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:56 smithi018 podman[70719]: 2024-09-21 17:29:56.003990237 +0000 UTC m=+0.932706885 container died 69f8a45e022d739898b7bc4fe679d43bd242694f469be6ad61ce302efa4cbf6f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T17:29:56.366 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:56 smithi018 ceph-mon[63924]: pgmap v89: 97 pgs: 10 active+undersized, 7 stale+active+clean, 11 active+undersized+degraded, 69 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.6 MiB/s wr, 443 op/s; 97/1698 objects degraded (5.713%) 2024-09-21T17:29:56.366 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:56 smithi018 ceph-mon[63924]: Health check failed: Degraded data redundancy: 97/1698 objects degraded (5.713%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 160637 8.92 MB/sec execute 301 sec latency 73.365 ms 2024-09-21T17:29:56.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:56 smithi157 ceph-mon[49460]: pgmap v89: 97 pgs: 10 active+undersized, 7 stale+active+clean, 11 active+undersized+degraded, 69 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.6 MiB/s wr, 443 op/s; 97/1698 objects degraded (5.713%) 2024-09-21T17:29:56.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:56 smithi157 ceph-mon[49460]: Health check failed: Degraded data redundancy: 97/1698 objects degraded (5.713%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T17:29:57.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:57 smithi018 podman[70742]: 2024-09-21 17:29:57.216890458 +0000 UTC m=+1.197778124 container remove 69f8a45e022d739898b7bc4fe679d43bd242694f469be6ad61ce302efa4cbf6f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T17:29:57.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:57 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T17:29:57.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:57 smithi018 systemd[1]: Stopped Ceph osd.1 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:29:57.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:57 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.1.service: Consumed 2min 33.324s CPU time. 2024-09-21T17:29:57.661 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:57 smithi018 systemd[1]: Starting Ceph osd.1 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:29:57.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 161024 8.93 MB/sec execute 302 sec latency 89.513 ms 2024-09-21T17:29:57.918 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:57 smithi018 podman[70837]: 2024-09-21 17:29:57.784631178 +0000 UTC m=+0.023730982 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:29:58.169 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:57 smithi018 podman[70837]: 2024-09-21 17:29:57.918013951 +0000 UTC m=+0.157113724 container create c06cdea15e118b7216631816d8fb9a2bee3ab3bfb4554295625ebb581ff4f4f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:29:58.421 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:58 smithi018 podman[70837]: 2024-09-21 17:29:58.314589683 +0000 UTC m=+0.553689484 container init c06cdea15e118b7216631816d8fb9a2bee3ab3bfb4554295625ebb581ff4f4f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2) 2024-09-21T17:29:58.421 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:58 smithi018 podman[70837]: 2024-09-21 17:29:58.328790503 +0000 UTC m=+0.567890279 container start c06cdea15e118b7216631816d8fb9a2bee3ab3bfb4554295625ebb581ff4f4f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T17:29:58.681 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:58 smithi018 podman[70837]: 2024-09-21 17:29:58.449970509 +0000 UTC m=+0.689070321 container attach c06cdea15e118b7216631816d8fb9a2bee3ab3bfb4554295625ebb581ff4f4f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T17:29:58.681 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:58 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:29:58.681 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:58 smithi018 bash[70837]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:29:58.681 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:58 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:29:58.681 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:58 smithi018 bash[70837]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:29:58.756 INFO:teuthology.orchestra.run.smithi018.stdout: 5 161357 8.91 MB/sec execute 303 sec latency 44.309 ms 2024-09-21T17:29:59.189 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:29:59 smithi018 ceph-mon[63924]: pgmap v90: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 546 op/s; 192/1665 objects degraded (11.532%) 2024-09-21T17:29:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:29:59 smithi157 ceph-mon[49460]: pgmap v90: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 546 op/s; 192/1665 objects degraded (11.532%) 2024-09-21T17:29:59.702 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:59 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T17:29:59.702 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:59 smithi018 bash[70837]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T17:29:59.702 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:59 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T17:29:59.703 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:29:59 smithi018 bash[70837]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T17:29:59.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 161609 8.90 MB/sec execute 304 sec latency 48.928 ms 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 bash[70837]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 bash[70837]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 bash[70837]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 bash[70837]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate[70852]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T17:30:00.211 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 bash[70837]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T17:30:00.212 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 podman[70837]: 2024-09-21 17:30:00.078409735 +0000 UTC m=+2.317509524 container died c06cdea15e118b7216631816d8fb9a2bee3ab3bfb4554295625ebb581ff4f4f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD) 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 192/1665 objects degraded (11.532%), 23 pgs degraded 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: osd.1 (root=default,host=smithi018) is down 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: [WRN] PG_DEGRADED: Degraded data redundancy: 192/1665 objects degraded (11.532%), 23 pgs degraded 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:00.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-21T17:30:00.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-21T17:30:00.214 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-21T17:30:00.214 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-21T17:30:00.214 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:00 smithi018 ceph-mon[63924]: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 192/1665 objects degraded (11.532%), 23 pgs degraded 2024-09-21T17:30:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T17:30:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: osd.1 (root=default,host=smithi018) is down 2024-09-21T17:30:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: [WRN] PG_DEGRADED: Degraded data redundancy: 192/1665 objects degraded (11.532%), 23 pgs degraded 2024-09-21T17:30:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-21T17:30:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:00.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:00.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-21T17:30:00.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-21T17:30:00.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-21T17:30:00.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-21T17:30:00.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-21T17:30:00.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-21T17:30:00.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:00 smithi157 ceph-mon[49460]: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:00.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 161900 8.90 MB/sec execute 305 sec latency 65.385 ms 2024-09-21T17:30:00.972 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:00 smithi018 podman[71002]: 2024-09-21 17:30:00.938868206 +0000 UTC m=+0.853742214 container remove c06cdea15e118b7216631816d8fb9a2bee3ab3bfb4554295625ebb581ff4f4f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-activate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:30:01.223 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:01 smithi018 ceph-mon[63924]: pgmap v91: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.3 MiB/s wr, 236 op/s; 192/1665 objects degraded (11.532%) 2024-09-21T17:30:01.223 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:01.223 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:30:01.224 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:01 smithi018 podman[71059]: 2024-09-21 17:30:01.08664831 +0000 UTC m=+0.019713649 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:30:01.224 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:01 smithi018 podman[71059]: 2024-09-21 17:30:01.206772404 +0000 UTC m=+0.139837733 container create 6ff6ff17584097eb1939543d20811f0b61edd63d2ee4ca4a239c826e797db44f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T17:30:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:01 smithi157 ceph-mon[49460]: pgmap v91: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.3 MiB/s wr, 236 op/s; 192/1665 objects degraded (11.532%) 2024-09-21T17:30:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:30:01.731 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:01 smithi018 podman[71059]: 2024-09-21 17:30:01.549908462 +0000 UTC m=+0.482973800 container init 6ff6ff17584097eb1939543d20811f0b61edd63d2ee4ca4a239c826e797db44f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T17:30:01.732 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:01 smithi018 podman[71059]: 2024-09-21 17:30:01.556909568 +0000 UTC m=+0.489974899 container start 6ff6ff17584097eb1939543d20811f0b61edd63d2ee4ca4a239c826e797db44f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:30:01.732 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:01 smithi018 ceph-osd[71093]: -- 172.21.15.18:0/2471344547 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55763b30c1e0 con 0x55763a4abc00 2024-09-21T17:30:01.732 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:01 smithi018 bash[71059]: 6ff6ff17584097eb1939543d20811f0b61edd63d2ee4ca4a239c826e797db44f 2024-09-21T17:30:01.732 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:01 smithi018 systemd[1]: Started Ceph osd.1 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:30:01.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 162324 8.93 MB/sec execute 306 sec latency 44.859 ms 2024-09-21T17:30:02.238 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:02 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 192/1665 objects degraded (11.532%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:02.239 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:02 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:02.239 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:02 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:02.239 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:02 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:30:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:02 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 192/1665 objects degraded (11.532%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:02 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:02 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:02 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:30:02.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 162696 8.94 MB/sec execute 307 sec latency 41.135 ms 2024-09-21T17:30:03.214 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:30:03.245 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:03 smithi018 ceph-mon[63924]: pgmap v92: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 472 op/s; 180/1677 objects degraded (10.733%) 2024-09-21T17:30:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:03 smithi157 ceph-mon[49460]: pgmap v92: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 472 op/s; 180/1677 objects degraded (10.733%) 2024-09-21T17:30:03.757 INFO:teuthology.orchestra.run.smithi018.stdout: 5 162990 8.95 MB/sec execute 308 sec latency 46.918 ms 2024-09-21T17:30:03.776 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:03 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[71089]: 2024-09-21T17:30:03.683+0000 7fb8a0fa7640 -1 Falling back to public interface 2024-09-21T17:30:04.278 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:04 smithi018 ceph-mon[63924]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (4m) 40s ago 13m 19.4M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (87s) 40s ago 13m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (81s) 75s ago 13m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (4m) 40s ago 13m 47.0M - dad864ee21e9 48d78bd31599 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (7m) 40s ago 7m 37.6M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (7m) 75s ago 7m 15.7M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (3m) 40s ago 15m 512M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (3m) 75s ago 13m 455M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (2m) 40s ago 15m 45.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (2m) 75s ago 13m 35.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (7m) 40s ago 7m 269M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:30:04.293 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (7m) 75s ago 7m 86.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (4m) 40s ago 13m 9935k - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (3m) 75s ago 12m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (49s) 40s ago 12m 11.1M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 starting - - - 1937M 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (11m) 40s ago 11m 359M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (10m) 40s ago 10m 484M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (10m) 75s ago 10m 483M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (10m) 75s ago 10m 536M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (9m) 75s ago 9m 456M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (9m) 75s ago 9m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:30:04.294 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (3m) 40s ago 12m 50.5M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:30:04.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:04 smithi157 ceph-mon[49460]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:04.759 INFO:teuthology.orchestra.run.smithi018.stdout: 5 163242 8.93 MB/sec execute 309 sec latency 42.908 ms 2024-09-21T17:30:04.913 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:30:04.913 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:30:04.913 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:30:04.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:30:04.913 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:30:04.913 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:30:04.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:30:04.913 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 5 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:30:04.914 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:30:05.304 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:05 smithi018 ceph-mon[63924]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:05.304 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:05 smithi018 ceph-mon[63924]: pgmap v93: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.9 MiB/s wr, 414 op/s; 180/1677 objects degraded (10.733%) 2024-09-21T17:30:05.304 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:05 smithi018 ceph-mon[63924]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:05.304 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:05 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3886810295' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "7/23 daemons upgraded", 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:30:05.428 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:30:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:05 smithi157 ceph-mon[49460]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:05 smithi157 ceph-mon[49460]: pgmap v93: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.9 MiB/s wr, 414 op/s; 180/1677 objects degraded (10.733%) 2024-09-21T17:30:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:05 smithi157 ceph-mon[49460]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:05 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3886810295' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:05.759 INFO:teuthology.orchestra.run.smithi018.stdout: 5 163534 8.91 MB/sec execute 310 sec latency 91.727 ms 2024-09-21T17:30:06.032 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 180/1677 objects degraded (10.733%), 23 pgs degraded 2024-09-21T17:30:06.032 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T17:30:06.032 INFO:teuthology.orchestra.run.smithi018.stdout: osd.1 (root=default,host=smithi018) is down 2024-09-21T17:30:06.032 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 180/1677 objects degraded (10.733%), 23 pgs degraded 2024-09-21T17:30:06.032 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-21T17:30:06.033 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:06.034 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-21T17:30:06.034 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-21T17:30:06.034 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-21T17:30:06.034 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-21T17:30:06.034 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-21T17:30:06.034 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-21T17:30:06.034 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-21T17:30:06.034 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-21T17:30:06.347 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:06 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3073669746' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:30:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:06 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3073669746' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:30:06.759 INFO:teuthology.orchestra.run.smithi018.stdout: 5 163776 8.90 MB/sec execute 311 sec latency 46.996 ms 2024-09-21T17:30:07.370 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:07 smithi018 ceph-mon[63924]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:07.371 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:07 smithi018 ceph-mon[63924]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 430 op/s; 188/1719 objects degraded (10.937%) 2024-09-21T17:30:07.371 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:07 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 188/1719 objects degraded (10.937%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:07 smithi157 ceph-mon[49460]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:07 smithi157 ceph-mon[49460]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 430 op/s; 188/1719 objects degraded (10.937%) 2024-09-21T17:30:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:07 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 188/1719 objects degraded (10.937%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:07.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 164081 8.90 MB/sec execute 312 sec latency 40.598 ms 2024-09-21T17:30:08.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 164457 8.91 MB/sec execute 313 sec latency 79.973 ms 2024-09-21T17:30:09.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:09 smithi018 ceph-mon[63924]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.6 MiB/s wr, 446 op/s; 193/1725 objects degraded (11.188%) 2024-09-21T17:30:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:09 smithi157 ceph-mon[49460]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.6 MiB/s wr, 446 op/s; 193/1725 objects degraded (11.188%) 2024-09-21T17:30:09.664 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:09 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[71089]: 2024-09-21T17:30:09.571+0000 7fb8a0fa7640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-21T17:30:09.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 164875 8.90 MB/sec execute 314 sec latency 73.024 ms 2024-09-21T17:30:10.695 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:10 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[71089]: 2024-09-21T17:30:10.593+0000 7fb8a0fa7640 -1 osd.1 85 log_to_monitors true 2024-09-21T17:30:10.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:10.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:10.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:10 smithi018 ceph-mon[63924]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 378 op/s; 193/1725 objects degraded (11.188%) 2024-09-21T17:30:10.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:10.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:10.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 165191 8.89 MB/sec execute 315 sec latency 36.269 ms 2024-09-21T17:30:11.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:11.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:11.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:10 smithi157 ceph-mon[49460]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 378 op/s; 193/1725 objects degraded (11.188%) 2024-09-21T17:30:11.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:11.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:11.703 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:30:11 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[71089]: 2024-09-21T17:30:11.627+0000 7fb898d51640 -1 osd.1 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:30:11.704 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:11 smithi018 ceph-mon[63924]: from='osd.1 [v2:172.21.15.18:6810/3058140136,v1:172.21.15.18:6811/3058140136]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T17:30:11.704 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:11 smithi018 ceph-mon[63924]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T17:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 165649 8.93 MB/sec execute 316 sec latency 31.965 ms 2024-09-21T17:30:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:11 smithi157 ceph-mon[49460]: from='osd.1 [v2:172.21.15.18:6810/3058140136,v1:172.21.15.18:6811/3058140136]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T17:30:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:11 smithi157 ceph-mon[49460]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T17:30:12.738 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:12 smithi018 ceph-mon[63924]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T17:30:12.738 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:12 smithi018 ceph-mon[63924]: from='osd.1 [v2:172.21.15.18:6810/3058140136,v1:172.21.15.18:6811/3058140136]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:30:12.738 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:12 smithi018 ceph-mon[63924]: osdmap e88: 8 total, 7 up, 8 in 2024-09-21T17:30:12.738 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:12 smithi018 ceph-mon[63924]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:30:12.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:12 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 193/1725 objects degraded (11.188%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:12.739 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:12 smithi018 ceph-mon[63924]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 475 op/s; 180/1668 objects degraded (10.791%) 2024-09-21T17:30:12.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 166089 8.94 MB/sec execute 317 sec latency 194.812 ms 2024-09-21T17:30:13.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:12 smithi157 ceph-mon[49460]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T17:30:13.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:12 smithi157 ceph-mon[49460]: from='osd.1 [v2:172.21.15.18:6810/3058140136,v1:172.21.15.18:6811/3058140136]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:30:13.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:12 smithi157 ceph-mon[49460]: osdmap e88: 8 total, 7 up, 8 in 2024-09-21T17:30:13.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:12 smithi157 ceph-mon[49460]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:30:13.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:12 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 193/1725 objects degraded (11.188%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:13.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:12 smithi157 ceph-mon[49460]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 475 op/s; 180/1668 objects degraded (10.791%) 2024-09-21T17:30:13.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 166089 8.91 MB/sec execute 318 sec latency 1194.845 ms 2024-09-21T17:30:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:13 smithi018 ceph-mon[63924]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:30:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:13 smithi018 ceph-mon[63924]: osd.1 [v2:172.21.15.18:6810/3058140136,v1:172.21.15.18:6811/3058140136] boot 2024-09-21T17:30:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:13 smithi018 ceph-mon[63924]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T17:30:13.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:13 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T17:30:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:13 smithi157 ceph-mon[49460]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:30:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:13 smithi157 ceph-mon[49460]: osd.1 [v2:172.21.15.18:6810/3058140136,v1:172.21.15.18:6811/3058140136] boot 2024-09-21T17:30:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:13 smithi157 ceph-mon[49460]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T17:30:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:13 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T17:30:14.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 166110 8.89 MB/sec execute 319 sec latency 2098.747 ms 2024-09-21T17:30:14.828 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:14 smithi018 ceph-mon[63924]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T17:30:14.828 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:14 smithi018 ceph-mon[63924]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.8 MiB/s wr, 411 op/s; 180/1668 objects degraded (10.791%) 2024-09-21T17:30:15.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:14 smithi157 ceph-mon[49460]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T17:30:15.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:14 smithi157 ceph-mon[49460]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.8 MiB/s wr, 411 op/s; 180/1668 objects degraded (10.791%) 2024-09-21T17:30:15.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:15 smithi018 ceph-mon[63924]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T17:30:15.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:15 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:15.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:15 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:30:15.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 166423 8.91 MB/sec execute 320 sec latency 203.353 ms 2024-09-21T17:30:16.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:15 smithi157 ceph-mon[49460]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T17:30:16.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:15 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:16.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:15 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:30:16.597 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: pgmap v104: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 8 active+undersized, 13 active+undersized+degraded, 71 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s wr, 114 op/s; 187/1632 objects degraded (11.458%); 1.0 MiB/s, 9 objects/s recovering 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:16.598 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T17:30:16.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 166454 8.88 MB/sec execute 321 sec latency 1038.025 ms 2024-09-21T17:30:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T17:30:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:30:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:30:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: pgmap v104: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 8 active+undersized, 13 active+undersized+degraded, 71 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s wr, 114 op/s; 187/1632 objects degraded (11.458%); 1.0 MiB/s, 9 objects/s recovering 2024-09-21T17:30:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:30:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T17:30:17.625 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:17 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T17:30:17.625 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:17 smithi018 ceph-mon[63924]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-21T17:30:17.625 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:17 smithi018 ceph-mon[63924]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T17:30:17.625 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:17 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 180/1668 objects degraded (10.791%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:17.626 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:17 smithi018 ceph-mon[63924]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T17:30:17.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 166661 8.86 MB/sec execute 322 sec latency 1004.956 ms 2024-09-21T17:30:18.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:17 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T17:30:18.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:17 smithi157 ceph-mon[49460]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-21T17:30:18.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:17 smithi157 ceph-mon[49460]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T17:30:18.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:17 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 180/1668 objects degraded (10.791%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:18.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:17 smithi157 ceph-mon[49460]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T17:30:18.661 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:18 smithi018 ceph-mon[63924]: pgmap v107: 97 pgs: 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 89 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 394 op/s; 97/1623 objects degraded (5.977%); 1.6 MiB/s, 64 keys/s, 39 objects/s recovering 2024-09-21T17:30:18.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 166867 8.84 MB/sec execute 323 sec latency 910.966 ms 2024-09-21T17:30:19.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:18 smithi157 ceph-mon[49460]: pgmap v107: 97 pgs: 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 89 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 394 op/s; 97/1623 objects degraded (5.977%); 1.6 MiB/s, 64 keys/s, 39 objects/s recovering 2024-09-21T17:30:19.689 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:19 smithi018 ceph-mon[63924]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T17:30:19.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 167158 8.83 MB/sec execute 324 sec latency 39.128 ms 2024-09-21T17:30:20.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:19 smithi157 ceph-mon[49460]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T17:30:20.630 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:20 smithi018 ceph-mon[63924]: pgmap v109: 97 pgs: 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 89 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 355 op/s; 97/1623 objects degraded (5.977%); 1.4 MiB/s, 58 keys/s, 36 objects/s recovering 2024-09-21T17:30:20.760 INFO:teuthology.orchestra.run.smithi018.stdout: 5 167500 8.82 MB/sec execute 325 sec latency 65.995 ms 2024-09-21T17:30:21.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:20 smithi157 ceph-mon[49460]: pgmap v109: 97 pgs: 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 89 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 355 op/s; 97/1623 objects degraded (5.977%); 1.4 MiB/s, 58 keys/s, 36 objects/s recovering 2024-09-21T17:30:21.765 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:21 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 97/1623 objects degraded (5.977%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:21.766 INFO:teuthology.orchestra.run.smithi018.stdout: 5 168023 8.84 MB/sec execute 326 sec latency 28.307 ms 2024-09-21T17:30:22.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:21 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 97/1623 objects degraded (5.977%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:22.766 INFO:teuthology.orchestra.run.smithi018.stdout: 5 168429 8.84 MB/sec execute 327 sec latency 31.016 ms 2024-09-21T17:30:22.786 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:22 smithi018 ceph-mon[63924]: pgmap v110: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 510 op/s; 377 KiB/s, 43 keys/s, 36 objects/s recovering 2024-09-21T17:30:23.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:22 smithi157 ceph-mon[49460]: pgmap v110: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 510 op/s; 377 KiB/s, 43 keys/s, 36 objects/s recovering 2024-09-21T17:30:23.766 INFO:teuthology.orchestra.run.smithi018.stdout: 5 168870 8.86 MB/sec execute 328 sec latency 48.845 ms 2024-09-21T17:30:23.815 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:23 smithi018 ceph-mon[63924]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 97/1623 objects degraded (5.977%), 4 pgs degraded) 2024-09-21T17:30:23.815 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:23 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:30:24.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:23 smithi157 ceph-mon[49460]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 97/1623 objects degraded (5.977%), 4 pgs degraded) 2024-09-21T17:30:24.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:23 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:30:24.766 INFO:teuthology.orchestra.run.smithi018.stdout: 5 169224 8.88 MB/sec execute 329 sec latency 48.625 ms 2024-09-21T17:30:24.849 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:24 smithi018 ceph-mon[63924]: pgmap v111: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.2 MiB/s wr, 261 op/s; 0 B/s, 13 objects/s recovering 2024-09-21T17:30:25.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:24 smithi157 ceph-mon[49460]: pgmap v111: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.2 MiB/s wr, 261 op/s; 0 B/s, 13 objects/s recovering 2024-09-21T17:30:25.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 169424 8.87 MB/sec execute 330 sec latency 66.652 ms 2024-09-21T17:30:26.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 169707 8.86 MB/sec execute 331 sec latency 74.048 ms 2024-09-21T17:30:27.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:27 smithi018 ceph-mon[63924]: pgmap v112: 97 pgs: 97 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.8 MiB/s wr, 514 op/s; 0 B/s, 11 objects/s recovering 2024-09-21T17:30:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:27 smithi157 ceph-mon[49460]: pgmap v112: 97 pgs: 97 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.8 MiB/s wr, 514 op/s; 0 B/s, 11 objects/s recovering 2024-09-21T17:30:27.766 INFO:teuthology.orchestra.run.smithi018.stdout: 5 170040 8.86 MB/sec execute 332 sec latency 47.825 ms 2024-09-21T17:30:28.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 170365 8.86 MB/sec execute 333 sec latency 64.936 ms 2024-09-21T17:30:29.200 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:29 smithi018 ceph-mon[63924]: pgmap v113: 97 pgs: 97 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 463 op/s; 0 B/s, 9 objects/s recovering 2024-09-21T17:30:29.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:29 smithi157 ceph-mon[49460]: pgmap v113: 97 pgs: 97 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 463 op/s; 0 B/s, 9 objects/s recovering 2024-09-21T17:30:29.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 170673 8.85 MB/sec execute 334 sec latency 52.946 ms 2024-09-21T17:30:30.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 170979 8.85 MB/sec execute 335 sec latency 49.950 ms 2024-09-21T17:30:31.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:31 smithi018 ceph-mon[63924]: pgmap v114: 97 pgs: 97 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 0 B/s, 8 objects/s recovering 2024-09-21T17:30:31.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:31.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:30:31.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:31 smithi157 ceph-mon[49460]: pgmap v114: 97 pgs: 97 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 0 B/s, 8 objects/s recovering 2024-09-21T17:30:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:30:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:31.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 171336 8.83 MB/sec execute 336 sec latency 54.142 ms 2024-09-21T17:30:32.251 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:32 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T17:30:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:32 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T17:30:32.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 171639 8.84 MB/sec execute 337 sec latency 61.268 ms 2024-09-21T17:30:33.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:33 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T17:30:33.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:33 smithi018 ceph-mon[63924]: Upgrade: osd.2 is safe to restart 2024-09-21T17:30:33.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:33 smithi018 ceph-mon[63924]: pgmap v115: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 599 op/s; 0 B/s, 8 objects/s recovering 2024-09-21T17:30:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:33 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T17:30:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:33 smithi157 ceph-mon[49460]: Upgrade: osd.2 is safe to restart 2024-09-21T17:30:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:33 smithi157 ceph-mon[49460]: pgmap v115: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 599 op/s; 0 B/s, 8 objects/s recovering 2024-09-21T17:30:33.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 171957 8.83 MB/sec execute 338 sec latency 47.182 ms 2024-09-21T17:30:34.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 172341 8.85 MB/sec execute 339 sec latency 51.840 ms 2024-09-21T17:30:34.798 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:34 smithi018 ceph-mon[63924]: Upgrade: Updating osd.2 2024-09-21T17:30:34.798 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:34 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:34.798 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:34 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T17:30:34.798 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:34 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:30:34.798 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:34 smithi018 ceph-mon[63924]: Deploying daemon osd.2 on smithi018 2024-09-21T17:30:34.798 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:34 smithi018 ceph-mon[63924]: pgmap v116: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 437 op/s 2024-09-21T17:30:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:34 smithi157 ceph-mon[49460]: Upgrade: Updating osd.2 2024-09-21T17:30:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:34 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:34 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T17:30:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:34 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:30:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:34 smithi157 ceph-mon[49460]: Deploying daemon osd.2 on smithi018 2024-09-21T17:30:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:34 smithi157 ceph-mon[49460]: pgmap v116: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 437 op/s 2024-09-21T17:30:35.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 172673 8.85 MB/sec execute 340 sec latency 41.512 ms 2024-09-21T17:30:36.616 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:30:36.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 172929 8.86 MB/sec execute 341 sec latency 65.604 ms 2024-09-21T17:30:37.109 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:37 smithi018 ceph-mon[63924]: pgmap v117: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 492 op/s 2024-09-21T17:30:37.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:37 smithi157 ceph-mon[49460]: pgmap v117: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 492 op/s 2024-09-21T17:30:37.702 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (5m) 28s ago 14m 19.5M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (2m) 28s ago 14m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (114s) 109s ago 13m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (5m) 28s ago 14m 47.3M - dad864ee21e9 48d78bd31599 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (8m) 28s ago 8m 38.0M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (8m) 109s ago 8m 15.7M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (4m) 28s ago 15m 516M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (3m) 109s ago 13m 455M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (3m) 28s ago 15m 48.1M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (2m) 109s ago 13m 35.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (7m) 28s ago 7m 273M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (7m) 109s ago 7m 86.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:30:37.703 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (4m) 28s ago 14m 9877k - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (4m) 109s ago 13m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (82s) 28s ago 12m 229M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (36s) 28s ago 12m 11.4M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (11m) 28s ago 11m 400M 1937M 17.2.0 e1d6a67b021e 4384081616da 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (11m) 28s ago 11m 513M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (11m) 109s ago 11m 483M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (10m) 109s ago 10m 536M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (10m) 109s ago 10m 456M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (9m) 109s ago 9m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:30:37.704 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (3m) 28s ago 13m 50.6M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:30:37.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 173191 8.85 MB/sec execute 342 sec latency 48.072 ms 2024-09-21T17:30:38.116 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:38 smithi018 ceph-mon[63924]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:38.117 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:38 smithi018 ceph-mon[63924]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:38.403 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:30:38.403 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:30:38.403 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:30:38.404 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-21T17:30:38.405 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 6 2024-09-21T17:30:38.405 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:30:38.405 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:30:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:38 smithi157 ceph-mon[49460]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:38 smithi157 ceph-mon[49460]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:38.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 173480 8.85 MB/sec execute 343 sec latency 93.666 ms 2024-09-21T17:30:38.986 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "8/23 daemons upgraded", 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:30:38.987 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:30:39.132 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:39 smithi018 ceph-mon[63924]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:39.133 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:39 smithi018 ceph-mon[63924]: pgmap v118: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.4 MiB/s wr, 424 op/s 2024-09-21T17:30:39.133 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:39 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2870636926' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:39.133 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:38 smithi018 systemd[1]: Stopping Ceph osd.2 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:30:39.384 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:39 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[40817]: 2024-09-21T17:30:39.374+0000 7f41914a8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:30:39.384 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:39 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[40817]: 2024-09-21T17:30:39.374+0000 7f41914a8700 -1 osd.2 95 *** Got signal Terminated *** 2024-09-21T17:30:39.385 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:39 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[40817]: 2024-09-21T17:30:39.374+0000 7f41914a8700 -1 osd.2 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:30:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:39 smithi157 ceph-mon[49460]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:39 smithi157 ceph-mon[49460]: pgmap v118: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.4 MiB/s wr, 424 op/s 2024-09-21T17:30:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:39 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2870636926' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:30:39.762 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:30:39.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 173842 8.85 MB/sec execute 344 sec latency 68.664 ms 2024-09-21T17:30:40.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:40 smithi018 ceph-mon[63924]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:40.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:40 smithi018 ceph-mon[63924]: osd.2 marked itself down and dead 2024-09-21T17:30:40.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:40 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3301366603' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:30:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:40 smithi157 ceph-mon[49460]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:30:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:40 smithi157 ceph-mon[49460]: osd.2 marked itself down and dead 2024-09-21T17:30:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:40 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3301366603' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:30:40.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 174054 8.84 MB/sec execute 345 sec latency 590.705 ms 2024-09-21T17:30:41.108 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:40 smithi018 podman[73102]: 2024-09-21 17:30:40.826566133 +0000 UTC m=+1.582430435 container died 4384081616da4a74c3bae10571703aaa894878796bfd9027b09b159f6cee64c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, vcs-type=git, architecture=x86_64, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.component=centos-stream-container, distribution-scope=public, ceph=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc.) 2024-09-21T17:30:41.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:41 smithi018 ceph-mon[63924]: pgmap v119: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 400 op/s 2024-09-21T17:30:41.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:41 smithi018 ceph-mon[63924]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:30:41.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:41 smithi018 ceph-mon[63924]: osdmap e96: 8 total, 7 up, 8 in 2024-09-21T17:30:41.360 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:41 smithi018 podman[73102]: 2024-09-21 17:30:41.231036991 +0000 UTC m=+1.986901296 container cleanup 4384081616da4a74c3bae10571703aaa894878796bfd9027b09b159f6cee64c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, name=centos-stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vcs-type=git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754) 2024-09-21T17:30:41.360 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:41 smithi018 bash[73102]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2 2024-09-21T17:30:41.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:41 smithi157 ceph-mon[49460]: pgmap v119: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 400 op/s 2024-09-21T17:30:41.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:41 smithi157 ceph-mon[49460]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:30:41.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:41 smithi157 ceph-mon[49460]: osdmap e96: 8 total, 7 up, 8 in 2024-09-21T17:30:41.618 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:41 smithi018 podman[73120]: 2024-09-21 17:30:41.558509622 +0000 UTC m=+0.733531095 container remove 4384081616da4a74c3bae10571703aaa894878796bfd9027b09b159f6cee64c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.tags=base centos centos-stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.openshift.expose-services=, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-21T17:30:41.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 174343 8.83 MB/sec execute 346 sec latency 1023.253 ms 2024-09-21T17:30:42.382 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:42 smithi018 ceph-mon[63924]: osdmap e97: 8 total, 7 up, 8 in 2024-09-21T17:30:42.382 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:42 smithi018 podman[73224]: 2024-09-21 17:30:42.069264588 +0000 UTC m=+0.023905889 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:30:42.383 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:42 smithi018 podman[73224]: 2024-09-21 17:30:42.285566415 +0000 UTC m=+0.240207707 container create 7538f379f9b668d9df6a32c0a32995762aae8a620873ccd6b96610b57f92d450 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T17:30:42.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:42 smithi157 ceph-mon[49460]: osdmap e97: 8 total, 7 up, 8 in 2024-09-21T17:30:42.767 INFO:teuthology.orchestra.run.smithi018.stdout: 5 174834 8.82 MB/sec execute 347 sec latency 40.744 ms 2024-09-21T17:30:42.903 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:42 smithi018 podman[73224]: 2024-09-21 17:30:42.664327935 +0000 UTC m=+0.618969240 container init 7538f379f9b668d9df6a32c0a32995762aae8a620873ccd6b96610b57f92d450 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:30:42.903 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:42 smithi018 podman[73224]: 2024-09-21 17:30:42.670839694 +0000 UTC m=+0.625480991 container start 7538f379f9b668d9df6a32c0a32995762aae8a620873ccd6b96610b57f92d450 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T17:30:42.903 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:42 smithi018 podman[73224]: 2024-09-21 17:30:42.848445149 +0000 UTC m=+0.803086442 container attach 7538f379f9b668d9df6a32c0a32995762aae8a620873ccd6b96610b57f92d450 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T17:30:43.158 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:42 smithi018 podman[73224]: 2024-09-21 17:30:42.976143855 +0000 UTC m=+0.930785153 container died 7538f379f9b668d9df6a32c0a32995762aae8a620873ccd6b96610b57f92d450 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:30:43.409 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:43 smithi018 ceph-mon[63924]: pgmap v122: 97 pgs: 2 active+undersized, 21 peering, 3 active+undersized+degraded, 71 active+clean; 363 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.7 MiB/s wr, 525 op/s; 29/1701 objects degraded (1.705%) 2024-09-21T17:30:43.409 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:43 smithi018 ceph-mon[63924]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-21T17:30:43.409 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:43 smithi018 ceph-mon[63924]: Health check failed: Degraded data redundancy: 29/1701 objects degraded (1.705%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:43 smithi157 ceph-mon[49460]: pgmap v122: 97 pgs: 2 active+undersized, 21 peering, 3 active+undersized+degraded, 71 active+clean; 363 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.7 MiB/s wr, 525 op/s; 29/1701 objects degraded (1.705%) 2024-09-21T17:30:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:43 smithi157 ceph-mon[49460]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-21T17:30:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:43 smithi157 ceph-mon[49460]: Health check failed: Degraded data redundancy: 29/1701 objects degraded (1.705%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: 5 175402 8.85 MB/sec execute 348 sec latency 32.518 ms 2024-09-21T17:30:44.162 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:44 smithi018 podman[73247]: 2024-09-21 17:30:44.15744961 +0000 UTC m=+1.173448618 container remove 7538f379f9b668d9df6a32c0a32995762aae8a620873ccd6b96610b57f92d450 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD) 2024-09-21T17:30:44.414 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:44 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T17:30:44.414 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:44 smithi018 systemd[1]: Stopped Ceph osd.2 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:30:44.414 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:44 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.2.service: Consumed 2min 5.802s CPU time. 2024-09-21T17:30:44.665 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:44 smithi018 systemd[1]: Starting Ceph osd.2 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:30:44.768 INFO:teuthology.orchestra.run.smithi018.stdout: 5 176011 8.88 MB/sec execute 349 sec latency 26.397 ms 2024-09-21T17:30:44.919 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:44 smithi018 podman[73340]: 2024-09-21 17:30:44.788878946 +0000 UTC m=+0.025128717 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:30:44.919 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:44 smithi018 podman[73340]: 2024-09-21 17:30:44.917203099 +0000 UTC m=+0.153452838 container create 1dd977f91a5757455401b590d9f4907047f7b4a1e8b750b948d7187a37be5473 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:30:45.422 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:45 smithi018 ceph-mon[63924]: pgmap v123: 97 pgs: 2 active+undersized, 21 peering, 3 active+undersized+degraded, 71 active+clean; 363 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 442 op/s; 29/1701 objects degraded (1.705%) 2024-09-21T17:30:45.423 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:45 smithi018 podman[73340]: 2024-09-21 17:30:45.304583806 +0000 UTC m=+0.540833556 container init 1dd977f91a5757455401b590d9f4907047f7b4a1e8b750b948d7187a37be5473 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:30:45.423 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:45 smithi018 podman[73340]: 2024-09-21 17:30:45.310531118 +0000 UTC m=+0.546780866 container start 1dd977f91a5757455401b590d9f4907047f7b4a1e8b750b948d7187a37be5473 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T17:30:45.423 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:45 smithi018 podman[73340]: 2024-09-21 17:30:45.412886079 +0000 UTC m=+0.649135819 container attach 1dd977f91a5757455401b590d9f4907047f7b4a1e8b750b948d7187a37be5473 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:30:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:45 smithi157 ceph-mon[49460]: pgmap v123: 97 pgs: 2 active+undersized, 21 peering, 3 active+undersized+degraded, 71 active+clean; 363 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 442 op/s; 29/1701 objects degraded (1.705%) 2024-09-21T17:30:45.667 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:45 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:30:45.667 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:45 smithi018 bash[73340]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:30:45.667 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:45 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:30:45.667 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:45 smithi018 bash[73340]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:30:45.768 INFO:teuthology.orchestra.run.smithi018.stdout: 5 176513 8.90 MB/sec execute 350 sec latency 35.952 ms 2024-09-21T17:30:46.679 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:46.679 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:30:46.679 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-mon[63924]: pgmap v124: 97 pgs: 6 active+undersized, 4 peering, 16 active+undersized+degraded, 71 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 5.5 MiB/s wr, 510 op/s; 139/1761 objects degraded (7.893%) 2024-09-21T17:30:46.679 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T17:30:46.679 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 bash[73340]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T17:30:46.679 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T17:30:46.679 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 bash[73340]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T17:30:46.768 INFO:teuthology.orchestra.run.smithi018.stdout: 5 177044 8.92 MB/sec execute 351 sec latency 29.185 ms 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 bash[73340]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 bash[73340]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 bash[73340]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 bash[73340]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate[73354]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 bash[73340]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T17:30:46.930 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:46 smithi018 podman[73340]: 2024-09-21 17:30:46.879751471 +0000 UTC m=+2.116001213 container died 1dd977f91a5757455401b590d9f4907047f7b4a1e8b750b948d7187a37be5473 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:30:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:30:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:46 smithi157 ceph-mon[49460]: pgmap v124: 97 pgs: 6 active+undersized, 4 peering, 16 active+undersized+degraded, 71 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 5.5 MiB/s wr, 510 op/s; 139/1761 objects degraded (7.893%) 2024-09-21T17:30:47.768 INFO:teuthology.orchestra.run.smithi018.stdout: 5 177564 8.93 MB/sec execute 352 sec latency 27.576 ms 2024-09-21T17:30:47.938 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:47 smithi018 podman[73488]: 2024-09-21 17:30:47.880047162 +0000 UTC m=+0.989728256 container remove 1dd977f91a5757455401b590d9f4907047f7b4a1e8b750b948d7187a37be5473 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T17:30:48.196 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:48 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 170/1815 objects degraded (9.366%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:48.196 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:48 smithi018 ceph-mon[63924]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T17:30:48.197 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:48 smithi018 podman[73540]: 2024-09-21 17:30:48.046361472 +0000 UTC m=+0.025192802 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:30:48.197 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:48 smithi018 podman[73540]: 2024-09-21 17:30:48.171988498 +0000 UTC m=+0.150819814 container create 3df4e3f0df26153c8f57f970bec2c865e4f905c85fc715ad29dd7407e3b073b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-21T17:30:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:48 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 170/1815 objects degraded (9.366%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:48 smithi157 ceph-mon[49460]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T17:30:48.768 INFO:teuthology.orchestra.run.smithi018.stdout: 5 178071 8.93 MB/sec execute 353 sec latency 41.433 ms 2024-09-21T17:30:49.223 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:49 smithi018 ceph-mon[63924]: pgmap v125: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.4 MiB/s wr, 704 op/s; 170/1815 objects degraded (9.366%) 2024-09-21T17:30:49.223 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:49 smithi018 podman[73540]: 2024-09-21 17:30:49.018622848 +0000 UTC m=+0.997454168 container init 3df4e3f0df26153c8f57f970bec2c865e4f905c85fc715ad29dd7407e3b073b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T17:30:49.223 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:49 smithi018 podman[73540]: 2024-09-21 17:30:49.03529489 +0000 UTC m=+1.014126205 container start 3df4e3f0df26153c8f57f970bec2c865e4f905c85fc715ad29dd7407e3b073b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T17:30:49.224 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:49 smithi018 ceph-osd[73556]: -- 172.21.15.18:0/312696979 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cb31c3a1e0 con 0x55cb31bc6000 2024-09-21T17:30:49.475 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:49 smithi018 bash[73540]: 3df4e3f0df26153c8f57f970bec2c865e4f905c85fc715ad29dd7407e3b073b7 2024-09-21T17:30:49.475 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:49 smithi018 systemd[1]: Started Ceph osd.2 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:30:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:49 smithi157 ceph-mon[49460]: pgmap v125: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.4 MiB/s wr, 704 op/s; 170/1815 objects degraded (9.366%) 2024-09-21T17:30:49.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 178545 8.93 MB/sec execute 354 sec latency 41.317 ms 2024-09-21T17:30:50.544 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:50 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:50.544 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:50 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:50.544 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:50 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:30:50.768 INFO:teuthology.orchestra.run.smithi018.stdout: 5 179204 8.97 MB/sec execute 355 sec latency 46.171 ms 2024-09-21T17:30:50.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:50 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:50.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:50 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:50.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:50 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:30:51.297 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:51 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[73552]: 2024-09-21T17:30:51.176+0000 7f464581d640 -1 Falling back to public interface 2024-09-21T17:30:51.548 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:51 smithi018 ceph-mon[63924]: pgmap v126: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.0 MiB/s wr, 571 op/s; 170/1815 objects degraded (9.366%) 2024-09-21T17:30:51.768 INFO:teuthology.orchestra.run.smithi018.stdout: 5 179889 9.01 MB/sec execute 356 sec latency 22.792 ms 2024-09-21T17:30:51.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:51 smithi157 ceph-mon[49460]: pgmap v126: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.0 MiB/s wr, 571 op/s; 170/1815 objects degraded (9.366%) 2024-09-21T17:30:52.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 180572 9.04 MB/sec execute 357 sec latency 28.484 ms 2024-09-21T17:30:53.312 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:53 smithi018 ceph-mon[63924]: pgmap v127: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 787 op/s; 160/1845 objects degraded (8.672%) 2024-09-21T17:30:53.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 181126 9.05 MB/sec execute 358 sec latency 49.532 ms 2024-09-21T17:30:53.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:53 smithi157 ceph-mon[49460]: pgmap v127: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 787 op/s; 160/1845 objects degraded (8.672%) 2024-09-21T17:30:54.587 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:54 smithi018 ceph-mon[63924]: pgmap v128: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.4 MiB/s wr, 631 op/s; 160/1845 objects degraded (8.672%) 2024-09-21T17:30:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 181467 9.05 MB/sec execute 359 sec latency 37.809 ms 2024-09-21T17:30:54.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:54 smithi157 ceph-mon[49460]: pgmap v128: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.4 MiB/s wr, 631 op/s; 160/1845 objects degraded (8.672%) 2024-09-21T17:30:55.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 181755 9.03 MB/sec execute 360 sec latency 67.952 ms 2024-09-21T17:30:56.357 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:56 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 159/1704 objects degraded (9.331%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:56 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 159/1704 objects degraded (9.331%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T17:30:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 182017 9.01 MB/sec execute 361 sec latency 64.481 ms 2024-09-21T17:30:57.140 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:56 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[73552]: 2024-09-21T17:30:56.984+0000 7f464581d640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-21T17:30:57.390 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:57 smithi018 ceph-mon[63924]: pgmap v129: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.3 MiB/s wr, 835 op/s; 159/1704 objects degraded (9.331%) 2024-09-21T17:30:57.429 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:57 smithi157 ceph-mon[49460]: pgmap v129: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.3 MiB/s wr, 835 op/s; 159/1704 objects degraded (9.331%) 2024-09-21T17:30:57.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 182530 9.04 MB/sec execute 362 sec latency 58.887 ms 2024-09-21T17:30:58.150 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:58 smithi018 ceph-mon[63924]: from='osd.2 [v2:172.21.15.18:6818/3889314548,v1:172.21.15.18:6819/3889314548]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T17:30:58.150 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:57 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[73552]: 2024-09-21T17:30:57.956+0000 7f464581d640 -1 osd.2 95 log_to_monitors true 2024-09-21T17:30:58.402 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:30:58 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[73552]: 2024-09-21T17:30:58.194+0000 7f463d5c7640 -1 osd.2 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:30:58.441 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:58 smithi157 ceph-mon[49460]: from='osd.2 [v2:172.21.15.18:6818/3889314548,v1:172.21.15.18:6819/3889314548]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T17:30:58.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 183217 9.07 MB/sec execute 363 sec latency 35.102 ms 2024-09-21T17:30:59.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:59 smithi018 ceph-mon[63924]: pgmap v130: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 7.4 MiB/s wr, 762 op/s; 155/1677 objects degraded (9.243%) 2024-09-21T17:30:59.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:59 smithi018 ceph-mon[63924]: from='osd.2 [v2:172.21.15.18:6818/3889314548,v1:172.21.15.18:6819/3889314548]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T17:30:59.170 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:59 smithi018 ceph-mon[63924]: osdmap e98: 8 total, 7 up, 8 in 2024-09-21T17:30:59.170 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:59 smithi018 ceph-mon[63924]: from='osd.2 [v2:172.21.15.18:6818/3889314548,v1:172.21.15.18:6819/3889314548]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:30:59.170 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:59 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:59.170 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:30:59 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:59.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:59 smithi157 ceph-mon[49460]: pgmap v130: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 7.4 MiB/s wr, 762 op/s; 155/1677 objects degraded (9.243%) 2024-09-21T17:30:59.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:59 smithi157 ceph-mon[49460]: from='osd.2 [v2:172.21.15.18:6818/3889314548,v1:172.21.15.18:6819/3889314548]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T17:30:59.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:59 smithi157 ceph-mon[49460]: osdmap e98: 8 total, 7 up, 8 in 2024-09-21T17:30:59.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:59 smithi157 ceph-mon[49460]: from='osd.2 [v2:172.21.15.18:6818/3889314548,v1:172.21.15.18:6819/3889314548]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:30:59.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:59 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:59.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:30:59 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:30:59.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 183446 9.08 MB/sec execute 364 sec latency 599.092 ms 2024-09-21T17:31:00.339 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:00 smithi018 ceph-mon[63924]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:31:00.340 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:00 smithi018 ceph-mon[63924]: osd.2 [v2:172.21.15.18:6818/3889314548,v1:172.21.15.18:6819/3889314548] boot 2024-09-21T17:31:00.340 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:00 smithi018 ceph-mon[63924]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T17:31:00.340 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T17:31:00.340 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:00.340 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:00 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:00 smithi157 ceph-mon[49460]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:31:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:00 smithi157 ceph-mon[49460]: osd.2 [v2:172.21.15.18:6818/3889314548,v1:172.21.15.18:6819/3889314548] boot 2024-09-21T17:31:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:00 smithi157 ceph-mon[49460]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T17:31:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T17:31:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:00 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:00.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 183446 9.05 MB/sec execute 365 sec latency 1599.125 ms 2024-09-21T17:31:01.392 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:01 smithi018 ceph-mon[63924]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 462 op/s; 155/1677 objects degraded (9.243%) 2024-09-21T17:31:01.392 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:01 smithi018 ceph-mon[63924]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T17:31:01.392 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:01.393 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:31:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:01 smithi157 ceph-mon[49460]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 462 op/s; 155/1677 objects degraded (9.243%) 2024-09-21T17:31:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:01 smithi157 ceph-mon[49460]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T17:31:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:31:01.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 183523 9.03 MB/sec execute 366 sec latency 2308.844 ms 2024-09-21T17:31:02.411 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:02 smithi018 ceph-mon[63924]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T17:31:02.411 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:02 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 155/1677 objects degraded (9.243%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:02.411 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:02 smithi018 ceph-mon[63924]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T17:31:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:02 smithi157 ceph-mon[49460]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T17:31:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:02 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 155/1677 objects degraded (9.243%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:02 smithi157 ceph-mon[49460]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T17:31:02.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 183842 9.03 MB/sec execute 367 sec latency 45.842 ms 2024-09-21T17:31:03.168 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:03 smithi018 ceph-mon[63924]: pgmap v137: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 8 remapped+peering, 1 active+undersized, 4 active+undersized+degraded, 81 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 526 B/s rd, 6.0 MiB/s wr, 533 op/s; 99/1638 objects degraded (6.044%); 0 B/s, 5 objects/s recovering 2024-09-21T17:31:03.168 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:03 smithi018 ceph-mon[63924]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T17:31:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:03 smithi157 ceph-mon[49460]: pgmap v137: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 8 remapped+peering, 1 active+undersized, 4 active+undersized+degraded, 81 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 526 B/s rd, 6.0 MiB/s wr, 533 op/s; 99/1638 objects degraded (6.044%); 0 B/s, 5 objects/s recovering 2024-09-21T17:31:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:03 smithi157 ceph-mon[49460]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T17:31:03.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 184150 9.03 MB/sec execute 368 sec latency 149.745 ms 2024-09-21T17:31:04.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 184331 9.02 MB/sec execute 369 sec latency 1026.671 ms 2024-09-21T17:31:04.888 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:04 smithi018 ceph-mon[63924]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T17:31:04.888 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:04 smithi018 ceph-mon[63924]: pgmap v140: 97 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 7 remapped+peering, 86 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.7 MiB/s wr, 665 op/s; 77/1641 objects degraded (4.692%); 41 KiB/s, 5 objects/s recovering 2024-09-21T17:31:05.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:04 smithi157 ceph-mon[49460]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T17:31:05.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:04 smithi157 ceph-mon[49460]: pgmap v140: 97 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 7 remapped+peering, 86 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.7 MiB/s wr, 665 op/s; 77/1641 objects degraded (4.692%); 41 KiB/s, 5 objects/s recovering 2024-09-21T17:31:05.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:05.657 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T17:31:05.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 184597 9.01 MB/sec execute 370 sec latency 54.245 ms 2024-09-21T17:31:06.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T17:31:06.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:06.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:06.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:31:06.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:31:06.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:06.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:06.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:31:06.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:06.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:06.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:06.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T17:31:06.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 184900 9.00 MB/sec execute 371 sec latency 58.623 ms 2024-09-21T17:31:06.932 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:06 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T17:31:06.932 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:06 smithi018 ceph-mon[63924]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-21T17:31:06.932 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:06 smithi018 ceph-mon[63924]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T17:31:06.932 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:06 smithi018 ceph-mon[63924]: pgmap v143: 97 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 93 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 340 op/s; 75/1713 objects degraded (4.378%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T17:31:06.933 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:06 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 77/1641 objects degraded (4.692%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:06 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T17:31:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:06 smithi157 ceph-mon[49460]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-21T17:31:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:06 smithi157 ceph-mon[49460]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T17:31:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:06 smithi157 ceph-mon[49460]: pgmap v143: 97 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 93 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 340 op/s; 75/1713 objects degraded (4.378%); 2.2 MiB/s, 35 objects/s recovering 2024-09-21T17:31:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:06 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 77/1641 objects degraded (4.692%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:07.716 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:07 smithi018 ceph-mon[63924]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T17:31:07.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 185323 9.00 MB/sec execute 372 sec latency 45.359 ms 2024-09-21T17:31:08.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:07 smithi157 ceph-mon[49460]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T17:31:08.769 INFO:teuthology.orchestra.run.smithi018.stdout: 5 185554 8.99 MB/sec execute 373 sec latency 72.634 ms 2024-09-21T17:31:08.778 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:08 smithi018 ceph-mon[63924]: pgmap v145: 97 pgs: 3 peering, 94 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s wr, 236 op/s; 2.1 MiB/s, 54 objects/s recovering 2024-09-21T17:31:09.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:08 smithi157 ceph-mon[49460]: pgmap v145: 97 pgs: 3 peering, 94 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s wr, 236 op/s; 2.1 MiB/s, 54 objects/s recovering 2024-09-21T17:31:09.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 186011 9.01 MB/sec execute 374 sec latency 74.522 ms 2024-09-21T17:31:09.805 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:09 smithi018 ceph-mon[63924]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/1713 objects degraded (4.378%), 2 pgs degraded) 2024-09-21T17:31:09.805 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:09 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:31:10.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:09 smithi157 ceph-mon[49460]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 75/1713 objects degraded (4.378%), 2 pgs degraded) 2024-09-21T17:31:10.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:09 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:31:10.413 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:31:10.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 186677 9.03 MB/sec execute 375 sec latency 27.144 ms 2024-09-21T17:31:10.811 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:10 smithi018 ceph-mon[63924]: pgmap v146: 97 pgs: 2 peering, 95 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.2 MiB/s wr, 246 op/s; 2.9 MiB/s, 40 objects/s recovering 2024-09-21T17:31:11.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:10 smithi157 ceph-mon[49460]: pgmap v146: 97 pgs: 2 peering, 95 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.2 MiB/s wr, 246 op/s; 2.9 MiB/s, 40 objects/s recovering 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (5m) 12s ago 15m 19.7M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (2m) 12s ago 14m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (2m) 2m ago 14m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (5m) 12s ago 14m 47.0M - dad864ee21e9 48d78bd31599 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (8m) 12s ago 8m 38.5M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (8m) 2m ago 8m 15.7M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (4m) 12s ago 16m 518M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (4m) 2m ago 14m 455M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (3m) 12s ago 16m 50.5M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (3m) 2m ago 14m 35.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (8m) 12s ago 8m 277M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:31:11.560 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (8m) 2m ago 8m 86.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (5m) 12s ago 14m 9831k - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (5m) 2m ago 14m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (116s) 12s ago 13m 280M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (70s) 12s ago 13m 224M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (23s) 12s ago 12m 14.4M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (12m) 12s ago 12m 561M 1937M 17.2.0 e1d6a67b021e c70f53907d1a 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (11m) 2m ago 11m 483M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (11m) 2m ago 11m 536M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (10m) 2m ago 10m 456M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (10m) 2m ago 10m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:31:11.561 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (4m) 12s ago 13m 52.9M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:31:11.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 187052 9.04 MB/sec execute 376 sec latency 61.556 ms 2024-09-21T17:31:11.818 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:11 smithi018 ceph-mon[63924]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:11.818 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:11 smithi018 ceph-mon[63924]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:11 smithi157 ceph-mon[49460]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:11 smithi157 ceph-mon[49460]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:12.188 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:31:12.188 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:31:12.188 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:31:12.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 3 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T17:31:12.189 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 7 2024-09-21T17:31:12.190 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:31:12.190 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:31:12.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 187446 9.05 MB/sec execute 377 sec latency 43.344 ms 2024-09-21T17:31:12.799 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:31:12.799 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "9/23 daemons upgraded", 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:31:12.800 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:31:12.846 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:12 smithi018 ceph-mon[63924]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:12.846 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:12 smithi018 ceph-mon[63924]: pgmap v147: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 6.4 MiB/s wr, 595 op/s; 2.4 MiB/s, 32 objects/s recovering 2024-09-21T17:31:12.846 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:12 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1683810569' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:12 smithi157 ceph-mon[49460]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:12 smithi157 ceph-mon[49460]: pgmap v147: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 6.4 MiB/s wr, 595 op/s; 2.4 MiB/s, 32 objects/s recovering 2024-09-21T17:31:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:12 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1683810569' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:13.549 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:31:13.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 187845 9.05 MB/sec execute 378 sec latency 39.037 ms 2024-09-21T17:31:13.852 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:13 smithi018 ceph-mon[63924]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:13.852 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:13 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/897970111' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:31:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:13 smithi157 ceph-mon[49460]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:13 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/897970111' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:31:14.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 188057 9.05 MB/sec execute 379 sec latency 54.623 ms 2024-09-21T17:31:14.888 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:14 smithi018 ceph-mon[63924]: pgmap v148: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 530 op/s; 2.1 MiB/s, 28 objects/s recovering 2024-09-21T17:31:15.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:14 smithi157 ceph-mon[49460]: pgmap v148: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 530 op/s; 2.1 MiB/s, 28 objects/s recovering 2024-09-21T17:31:15.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 188309 9.04 MB/sec execute 380 sec latency 56.237 ms 2024-09-21T17:31:16.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:16.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:31:16.658 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:16 smithi018 ceph-mon[63924]: pgmap v149: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 479 op/s; 897 KiB/s, 10 objects/s recovering 2024-09-21T17:31:16.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 188597 9.02 MB/sec execute 381 sec latency 89.944 ms 2024-09-21T17:31:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:31:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:16 smithi157 ceph-mon[49460]: pgmap v149: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 479 op/s; 897 KiB/s, 10 objects/s recovering 2024-09-21T17:31:17.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 188906 9.01 MB/sec execute 382 sec latency 48.144 ms 2024-09-21T17:31:18.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 189262 9.01 MB/sec execute 383 sec latency 41.209 ms 2024-09-21T17:31:19.239 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:19 smithi018 ceph-mon[63924]: pgmap v150: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 511 op/s; 722 KiB/s, 0 objects/s recovering 2024-09-21T17:31:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:19 smithi157 ceph-mon[49460]: pgmap v150: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 511 op/s; 722 KiB/s, 0 objects/s recovering 2024-09-21T17:31:19.770 INFO:teuthology.orchestra.run.smithi018.stdout: 5 189592 9.02 MB/sec execute 384 sec latency 45.583 ms 2024-09-21T17:31:20.774 INFO:teuthology.orchestra.run.smithi018.stdout: 5 189960 9.03 MB/sec execute 385 sec latency 59.513 ms 2024-09-21T17:31:21.170 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:21 smithi018 ceph-mon[63924]: pgmap v151: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.7 MiB/s wr, 507 op/s; 683 KiB/s, 0 objects/s recovering 2024-09-21T17:31:21.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T17:31:21.171 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:21 smithi157 ceph-mon[49460]: pgmap v151: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.7 MiB/s wr, 507 op/s; 683 KiB/s, 0 objects/s recovering 2024-09-21T17:31:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T17:31:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:21.774 INFO:teuthology.orchestra.run.smithi018.stdout: 5 190343 9.03 MB/sec execute 386 sec latency 39.405 ms 2024-09-21T17:31:22.177 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:22 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T17:31:22.177 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:22 smithi018 ceph-mon[63924]: Upgrade: osd.3 is safe to restart 2024-09-21T17:31:22.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:22 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T17:31:22.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:22 smithi157 ceph-mon[49460]: Upgrade: osd.3 is safe to restart 2024-09-21T17:31:22.774 INFO:teuthology.orchestra.run.smithi018.stdout: 5 190774 9.05 MB/sec execute 387 sec latency 30.114 ms 2024-09-21T17:31:23.193 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:23 smithi018 ceph-mon[63924]: pgmap v152: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.0 MiB/s wr, 575 op/s 2024-09-21T17:31:23.193 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:23.193 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T17:31:23.194 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:31:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:23 smithi157 ceph-mon[49460]: pgmap v152: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.0 MiB/s wr, 575 op/s 2024-09-21T17:31:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T17:31:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:31:23.774 INFO:teuthology.orchestra.run.smithi018.stdout: 5 191162 9.05 MB/sec execute 388 sec latency 41.483 ms 2024-09-21T17:31:24.238 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:24 smithi018 ceph-mon[63924]: Upgrade: Updating osd.3 2024-09-21T17:31:24.239 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:24 smithi018 ceph-mon[63924]: Deploying daemon osd.3 on smithi018 2024-09-21T17:31:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:24 smithi157 ceph-mon[49460]: Upgrade: Updating osd.3 2024-09-21T17:31:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:24 smithi157 ceph-mon[49460]: Deploying daemon osd.3 on smithi018 2024-09-21T17:31:24.774 INFO:teuthology.orchestra.run.smithi018.stdout: 5 191476 9.05 MB/sec execute 389 sec latency 56.269 ms 2024-09-21T17:31:25.264 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:25 smithi018 ceph-mon[63924]: pgmap v153: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 469 op/s 2024-09-21T17:31:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:25 smithi157 ceph-mon[49460]: pgmap v153: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 469 op/s 2024-09-21T17:31:25.774 INFO:teuthology.orchestra.run.smithi018.stdout: 5 191726 9.04 MB/sec execute 390 sec latency 74.450 ms 2024-09-21T17:31:26.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 192053 9.03 MB/sec execute 391 sec latency 96.915 ms 2024-09-21T17:31:27.336 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:27 smithi018 ceph-mon[63924]: pgmap v154: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.8 MiB/s wr, 576 op/s 2024-09-21T17:31:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:27 smithi157 ceph-mon[49460]: pgmap v154: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.8 MiB/s wr, 576 op/s 2024-09-21T17:31:27.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 192519 9.03 MB/sec execute 392 sec latency 66.898 ms 2024-09-21T17:31:28.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 192822 9.03 MB/sec execute 393 sec latency 79.752 ms 2024-09-21T17:31:29.155 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:29 smithi018 ceph-mon[63924]: pgmap v155: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 495 op/s 2024-09-21T17:31:29.156 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:29 smithi018 systemd[1]: Stopping Ceph osd.3 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:31:29.412 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:29 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[44685]: 2024-09-21T17:31:29.383+0000 7f3f0d35b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:31:29.412 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:29 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[44685]: 2024-09-21T17:31:29.383+0000 7f3f0d35b700 -1 osd.3 107 *** Got signal Terminated *** 2024-09-21T17:31:29.413 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:29 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[44685]: 2024-09-21T17:31:29.383+0000 7f3f0d35b700 -1 osd.3 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:31:29.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:29 smithi157 ceph-mon[49460]: pgmap v155: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 495 op/s 2024-09-21T17:31:29.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 193106 9.03 MB/sec execute 394 sec latency 102.242 ms 2024-09-21T17:31:30.204 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:30 smithi018 ceph-mon[63924]: osd.3 marked itself down and dead 2024-09-21T17:31:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:30 smithi157 ceph-mon[49460]: osd.3 marked itself down and dead 2024-09-21T17:31:30.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 193271 9.02 MB/sec execute 395 sec latency 627.475 ms 2024-09-21T17:31:31.136 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:30 smithi018 podman[75412]: 2024-09-21 17:31:30.869360819 +0000 UTC m=+1.623875153 container died c70f53907d1a5b6b29e061d2904049f0b0e42486af592231d640d487505b9cc1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , ceph=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.expose-services=, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8) 2024-09-21T17:31:31.393 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:31 smithi018 ceph-mon[63924]: pgmap v156: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 4.3 MiB/s wr, 420 op/s 2024-09-21T17:31:31.394 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:31 smithi018 ceph-mon[63924]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:31:31.394 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:31 smithi018 ceph-mon[63924]: osdmap e108: 8 total, 7 up, 8 in 2024-09-21T17:31:31.394 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:31.394 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:31:31.394 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:31 smithi018 podman[75412]: 2024-09-21 17:31:31.285488792 +0000 UTC m=+2.040003114 container cleanup c70f53907d1a5b6b29e061d2904049f0b0e42486af592231d640d487505b9cc1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, name=centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, ceph=True) 2024-09-21T17:31:31.394 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:31 smithi018 bash[75412]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3 2024-09-21T17:31:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:31 smithi157 ceph-mon[49460]: pgmap v156: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 4.3 MiB/s wr, 420 op/s 2024-09-21T17:31:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:31 smithi157 ceph-mon[49460]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:31:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:31 smithi157 ceph-mon[49460]: osdmap e108: 8 total, 7 up, 8 in 2024-09-21T17:31:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:31:31.645 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:31 smithi018 podman[75428]: 2024-09-21 17:31:31.568872928 +0000 UTC m=+0.702137514 container remove c70f53907d1a5b6b29e061d2904049f0b0e42486af592231d640d487505b9cc1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, release=754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , distribution-scope=public, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-type=git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vendor=Red Hat, Inc., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-21T17:31:31.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 193503 9.02 MB/sec execute 396 sec latency 1031.776 ms 2024-09-21T17:31:32.153 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:32 smithi018 podman[75515]: 2024-09-21 17:31:32.016219948 +0000 UTC m=+0.025258145 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:31:32.153 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:32 smithi018 podman[75515]: 2024-09-21 17:31:32.130686275 +0000 UTC m=+0.139724463 container create 49ef5f5b51691e3daa6edf5a8936e810a62a671f42653ae5bbe2a8da57f59813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:31:32.416 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:32 smithi018 ceph-mon[63924]: osdmap e109: 8 total, 7 up, 8 in 2024-09-21T17:31:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:32 smithi157 ceph-mon[49460]: osdmap e109: 8 total, 7 up, 8 in 2024-09-21T17:31:32.668 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:32 smithi018 podman[75515]: 2024-09-21 17:31:32.492940251 +0000 UTC m=+0.501978447 container init 49ef5f5b51691e3daa6edf5a8936e810a62a671f42653ae5bbe2a8da57f59813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:31:32.668 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:32 smithi018 podman[75515]: 2024-09-21 17:31:32.504938974 +0000 UTC m=+0.513977170 container start 49ef5f5b51691e3daa6edf5a8936e810a62a671f42653ae5bbe2a8da57f59813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T17:31:32.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 193836 9.01 MB/sec execute 397 sec latency 45.265 ms 2024-09-21T17:31:32.921 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:32 smithi018 podman[75515]: 2024-09-21 17:31:32.666954889 +0000 UTC m=+0.675993077 container attach 49ef5f5b51691e3daa6edf5a8936e810a62a671f42653ae5bbe2a8da57f59813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T17:31:32.921 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:32 smithi018 podman[75515]: 2024-09-21 17:31:32.801647268 +0000 UTC m=+0.810685471 container died 49ef5f5b51691e3daa6edf5a8936e810a62a671f42653ae5bbe2a8da57f59813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T17:31:33.429 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:33 smithi018 ceph-mon[63924]: pgmap v159: 97 pgs: 4 active+undersized, 27 peering, 1 stale+active+clean, 6 active+undersized+degraded, 59 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 MiB/s wr, 502 op/s; 28/1572 objects degraded (1.781%) 2024-09-21T17:31:33.429 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:33 smithi018 ceph-mon[63924]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-21T17:31:33.429 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:33 smithi018 ceph-mon[63924]: Health check failed: Degraded data redundancy: 28/1572 objects degraded (1.781%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:33 smithi157 ceph-mon[49460]: pgmap v159: 97 pgs: 4 active+undersized, 27 peering, 1 stale+active+clean, 6 active+undersized+degraded, 59 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 MiB/s wr, 502 op/s; 28/1572 objects degraded (1.781%) 2024-09-21T17:31:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:33 smithi157 ceph-mon[49460]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-21T17:31:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:33 smithi157 ceph-mon[49460]: Health check failed: Degraded data redundancy: 28/1572 objects degraded (1.781%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:33.680 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:33 smithi018 podman[75538]: 2024-09-21 17:31:33.678237258 +0000 UTC m=+0.859611910 container remove 49ef5f5b51691e3daa6edf5a8936e810a62a671f42653ae5bbe2a8da57f59813 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T17:31:33.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 194089 9.02 MB/sec execute 398 sec latency 48.679 ms 2024-09-21T17:31:33.954 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:33 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T17:31:33.954 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:33 smithi018 systemd[1]: Stopped Ceph osd.3 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:31:33.954 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:33 smithi018 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.3.service: Consumed 3min 45.919s CPU time. 2024-09-21T17:31:34.211 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 systemd[1]: Starting Ceph osd.3 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:31:34.474 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 podman[75634]: 2024-09-21 17:31:34.207405612 +0000 UTC m=+0.018105394 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:31:34.474 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 podman[75634]: 2024-09-21 17:31:34.40222678 +0000 UTC m=+0.212926554 container create 0b1fdf7d8f554e72f4d3575a9c7314bbb5ab38c0fb43ca88bd80e004248613f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True) 2024-09-21T17:31:34.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 194385 9.02 MB/sec execute 399 sec latency 44.431 ms 2024-09-21T17:31:34.986 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 podman[75634]: 2024-09-21 17:31:34.734148096 +0000 UTC m=+0.544847880 container init 0b1fdf7d8f554e72f4d3575a9c7314bbb5ab38c0fb43ca88bd80e004248613f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:31:34.987 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 podman[75634]: 2024-09-21 17:31:34.743111239 +0000 UTC m=+0.553811018 container start 0b1fdf7d8f554e72f4d3575a9c7314bbb5ab38c0fb43ca88bd80e004248613f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T17:31:34.987 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:31:34.987 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:31:34.987 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 bash[75634]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:31:34.987 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 bash[75634]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:31:34.987 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:34 smithi018 podman[75634]: 2024-09-21 17:31:34.977262063 +0000 UTC m=+0.787961839 container attach 0b1fdf7d8f554e72f4d3575a9c7314bbb5ab38c0fb43ca88bd80e004248613f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:31:35.245 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:35 smithi018 ceph-mon[63924]: pgmap v160: 97 pgs: 5 active+undersized, 27 peering, 9 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.4 MiB/s wr, 342 op/s; 40/1572 objects degraded (2.545%) 2024-09-21T17:31:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:35 smithi157 ceph-mon[49460]: pgmap v160: 97 pgs: 5 active+undersized, 27 peering, 9 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.4 MiB/s wr, 342 op/s; 40/1572 objects degraded (2.545%) 2024-09-21T17:31:35.768 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T17:31:35.768 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 bash[75634]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T17:31:35.768 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T17:31:35.769 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 bash[75634]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T17:31:35.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 194648 9.01 MB/sec execute 400 sec latency 76.813 ms 2024-09-21T17:31:36.022 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T17:31:36.022 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 bash[75634]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T17:31:36.022 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T17:31:36.022 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 bash[75634]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T17:31:36.022 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T17:31:36.023 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 bash[75634]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T17:31:36.023 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T17:31:36.023 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 bash[75634]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T17:31:36.023 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate[75649]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T17:31:36.023 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:35 smithi018 bash[75634]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T17:31:36.023 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:36 smithi018 podman[75634]: 2024-09-21 17:31:36.00804325 +0000 UTC m=+1.818743033 container died 0b1fdf7d8f554e72f4d3575a9c7314bbb5ab38c0fb43ca88bd80e004248613f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T17:31:36.272 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:36 smithi018 ceph-mon[63924]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-21T17:31:36.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:36 smithi157 ceph-mon[49460]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-21T17:31:36.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 195010 9.02 MB/sec execute 401 sec latency 75.758 ms 2024-09-21T17:31:37.317 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:37 smithi018 ceph-mon[63924]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 429 op/s; 208/1698 objects degraded (12.250%) 2024-09-21T17:31:37.318 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:37 smithi018 podman[75795]: 2024-09-21 17:31:37.066897613 +0000 UTC m=+1.052034732 container remove 0b1fdf7d8f554e72f4d3575a9c7314bbb5ab38c0fb43ca88bd80e004248613f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T17:31:37.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:37 smithi157 ceph-mon[49460]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 429 op/s; 208/1698 objects degraded (12.250%) 2024-09-21T17:31:37.578 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:37 smithi018 podman[75852]: 2024-09-21 17:31:37.225690443 +0000 UTC m=+0.025225590 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:31:37.578 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:37 smithi018 podman[75852]: 2024-09-21 17:31:37.362318417 +0000 UTC m=+0.161853547 container create e7c60a62ac9e5c14edb41462112f5cf3b1b4445867afd785f8704420af92846f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T17:31:37.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 195286 9.01 MB/sec execute 402 sec latency 62.090 ms 2024-09-21T17:31:37.829 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:37 smithi018 podman[75852]: 2024-09-21 17:31:37.731261748 +0000 UTC m=+0.530796889 container init e7c60a62ac9e5c14edb41462112f5cf3b1b4445867afd785f8704420af92846f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T17:31:37.829 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:37 smithi018 podman[75852]: 2024-09-21 17:31:37.742392033 +0000 UTC m=+0.541927168 container start e7c60a62ac9e5c14edb41462112f5cf3b1b4445867afd785f8704420af92846f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:31:37.829 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:37 smithi018 ceph-osd[75870]: -- 172.21.15.18:0/4036022123 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bb4c29a1e0 con 0x55bb4b4a5c00 2024-09-21T17:31:38.087 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:37 smithi018 bash[75852]: e7c60a62ac9e5c14edb41462112f5cf3b1b4445867afd785f8704420af92846f 2024-09-21T17:31:38.087 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:37 smithi018 systemd[1]: Started Ceph osd.3 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:31:38.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 195603 9.00 MB/sec execute 403 sec latency 59.078 ms 2024-09-21T17:31:39.094 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:38 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:39.094 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:38 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:39.094 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:38 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:31:39.094 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:38 smithi018 ceph-mon[63924]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s wr, 527 op/s; 208/1722 objects degraded (12.079%) 2024-09-21T17:31:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:38 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:38 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:38 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:31:39.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:38 smithi157 ceph-mon[49460]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s wr, 527 op/s; 208/1722 objects degraded (12.079%) 2024-09-21T17:31:39.345 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:39 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:31:39.110+0000 7f7053f70640 -1 Falling back to public interface 2024-09-21T17:31:39.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 195883 8.99 MB/sec execute 404 sec latency 79.386 ms 2024-09-21T17:31:40.124 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:39 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 208/1722 objects degraded (12.079%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:40.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:39 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 208/1722 objects degraded (12.079%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:40.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 196163 8.98 MB/sec execute 405 sec latency 77.898 ms 2024-09-21T17:31:41.138 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:40 smithi018 ceph-mon[63924]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.4 MiB/s wr, 427 op/s; 208/1722 objects degraded (12.079%) 2024-09-21T17:31:41.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:40 smithi157 ceph-mon[49460]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.4 MiB/s wr, 427 op/s; 208/1722 objects degraded (12.079%) 2024-09-21T17:31:41.775 INFO:teuthology.orchestra.run.smithi018.stdout: 5 196497 8.99 MB/sec execute 406 sec latency 46.527 ms 2024-09-21T17:31:42.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 196961 9.01 MB/sec execute 407 sec latency 47.299 ms 2024-09-21T17:31:43.177 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:43 smithi018 ceph-mon[63924]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.3 MiB/s wr, 411 op/s; 210/1704 objects degraded (12.324%) 2024-09-21T17:31:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:43 smithi157 ceph-mon[49460]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.3 MiB/s wr, 411 op/s; 210/1704 objects degraded (12.324%) 2024-09-21T17:31:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 197342 9.00 MB/sec execute 408 sec latency 46.380 ms 2024-09-21T17:31:44.182 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:31:44.223 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:44 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 210/1701 objects degraded (12.346%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:44 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 210/1701 objects degraded (12.346%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:44.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 197569 9.00 MB/sec execute 409 sec latency 54.434 ms 2024-09-21T17:31:45.263 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:45 smithi018 ceph-mon[63924]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.7 MiB/s wr, 366 op/s; 210/1701 objects degraded (12.346%) 2024-09-21T17:31:45.264 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:45 smithi018 ceph-mon[63924]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:45.337 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:31:45.337 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (6m) 46s ago 15m 19.7M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:31:45.337 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (3m) 46s ago 15m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:31:45.337 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (3m) 2m ago 14m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:31:45.337 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (6m) 46s ago 15m 47.0M - dad864ee21e9 48d78bd31599 2024-09-21T17:31:45.337 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (9m) 46s ago 9m 38.5M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:31:45.337 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (9m) 2m ago 9m 15.7M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:31:45.337 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (5m) 46s ago 16m 518M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (4m) 2m ago 14m 455M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (4m) 46s ago 16m 50.5M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (3m) 2m ago 14m 35.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (9m) 46s ago 9m 277M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (8m) 2m ago 8m 86.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (6m) 46s ago 15m 9831k - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (5m) 2m ago 14m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (2m) 46s ago 14m 280M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (104s) 46s ago 13m 224M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (57s) 46s ago 13m 14.4M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 starting - - - 1937M 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (12m) 2m ago 12m 483M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (11m) 2m ago 11m 536M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:31:45.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (11m) 2m ago 11m 456M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:31:45.339 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (10m) 2m ago 10m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:31:45.339 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (4m) 46s ago 14m 52.9M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:31:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:45 smithi157 ceph-mon[49460]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.7 MiB/s wr, 366 op/s; 210/1701 objects degraded (12.346%) 2024-09-21T17:31:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:45 smithi157 ceph-mon[49460]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:45.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 197690 8.99 MB/sec execute 410 sec latency 62.323 ms 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 3 2024-09-21T17:31:46.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:31:46.004 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:31:46.004 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:31:46.004 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:31:46.004 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:31:46.004 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T17:31:46.004 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 7 2024-09-21T17:31:46.004 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:31:46.004 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:31:46.058 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:45 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:31:45.854+0000 7f7053f70640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-21T17:31:46.309 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:46 smithi018 ceph-mon[63924]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:46.310 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:46.310 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:31:46.310 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:46.310 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:46.310 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:46 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2813806859' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:46 smithi157 ceph-mon[49460]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:31:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:46 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2813806859' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:46.599 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "10/23 daemons upgraded", 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:31:46.600 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:31:46.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 197841 8.98 MB/sec execute 411 sec latency 79.836 ms 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 204/1650 objects degraded (12.364%), 27 pgs degraded 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout: osd.3 (root=default,host=smithi018) is down 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 204/1650 objects degraded (12.364%), 27 pgs degraded 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-09-21T17:31:47.255 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-21T17:31:47.256 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-09-21T17:31:47.257 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-21T17:31:47.339 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:47 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:31:47.188+0000 7f7053f70640 -1 osd.3 107 log_to_monitors true 2024-09-21T17:31:47.339 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:47 smithi018 ceph-mon[63924]: from='client.34318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:47.339 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:47 smithi018 ceph-mon[63924]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.5 MiB/s wr, 534 op/s; 204/1650 objects degraded (12.364%) 2024-09-21T17:31:47.340 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:47 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:47.340 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:47 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:47 smithi157 ceph-mon[49460]: from='client.34318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:47 smithi157 ceph-mon[49460]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.5 MiB/s wr, 534 op/s; 204/1650 objects degraded (12.364%) 2024-09-21T17:31:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:47 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:47 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:47.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 198085 8.98 MB/sec execute 412 sec latency 48.385 ms 2024-09-21T17:31:48.358 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:48 smithi018 ceph-mon[63924]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:48.358 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:48 smithi018 ceph-mon[63924]: from='osd.3 [v2:172.21.15.18:6826/2366666771,v1:172.21.15.18:6827/2366666771]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T17:31:48.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:48 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3760340873' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:31:48.359 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:31:48 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:31:48.243+0000 7f704bd1a640 -1 osd.3 107 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:31:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:48 smithi157 ceph-mon[49460]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:31:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:48 smithi157 ceph-mon[49460]: from='osd.3 [v2:172.21.15.18:6826/2366666771,v1:172.21.15.18:6827/2366666771]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T17:31:48.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:48 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3760340873' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:31:48.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 198383 8.97 MB/sec execute 413 sec latency 59.868 ms 2024-09-21T17:31:49.364 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:49 smithi018 ceph-mon[63924]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 464 op/s; 202/1644 objects degraded (12.287%) 2024-09-21T17:31:49.365 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:49 smithi018 ceph-mon[63924]: from='osd.3 [v2:172.21.15.18:6826/2366666771,v1:172.21.15.18:6827/2366666771]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T17:31:49.365 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:49 smithi018 ceph-mon[63924]: osdmap e110: 8 total, 7 up, 8 in 2024-09-21T17:31:49.365 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:49 smithi018 ceph-mon[63924]: from='osd.3 [v2:172.21.15.18:6826/2366666771,v1:172.21.15.18:6827/2366666771]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:31:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:49 smithi157 ceph-mon[49460]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 464 op/s; 202/1644 objects degraded (12.287%) 2024-09-21T17:31:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:49 smithi157 ceph-mon[49460]: from='osd.3 [v2:172.21.15.18:6826/2366666771,v1:172.21.15.18:6827/2366666771]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T17:31:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:49 smithi157 ceph-mon[49460]: osdmap e110: 8 total, 7 up, 8 in 2024-09-21T17:31:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:49 smithi157 ceph-mon[49460]: from='osd.3 [v2:172.21.15.18:6826/2366666771,v1:172.21.15.18:6827/2366666771]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]} : dispatch 2024-09-21T17:31:49.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 198595 8.97 MB/sec execute 414 sec latency 599.417 ms 2024-09-21T17:31:50.448 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:50 smithi018 ceph-mon[63924]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:31:50.448 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:50 smithi018 ceph-mon[63924]: osd.3 [v2:172.21.15.18:6826/2366666771,v1:172.21.15.18:6827/2366666771] boot 2024-09-21T17:31:50.448 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:50 smithi018 ceph-mon[63924]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T17:31:50.448 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:50 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T17:31:50.448 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:50 smithi018 ceph-mon[63924]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T17:31:50.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:50 smithi157 ceph-mon[49460]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:31:50.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:50 smithi157 ceph-mon[49460]: osd.3 [v2:172.21.15.18:6826/2366666771,v1:172.21.15.18:6827/2366666771] boot 2024-09-21T17:31:50.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:50 smithi157 ceph-mon[49460]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T17:31:50.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:50 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T17:31:50.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:50 smithi157 ceph-mon[49460]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T17:31:50.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 198668 8.95 MB/sec execute 415 sec latency 1599.448 ms 2024-09-21T17:31:51.463 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:51 smithi018 ceph-mon[63924]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.0 MiB/s wr, 273 op/s; 202/1644 objects degraded (12.287%) 2024-09-21T17:31:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:51 smithi157 ceph-mon[49460]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.0 MiB/s wr, 273 op/s; 202/1644 objects degraded (12.287%) 2024-09-21T17:31:51.776 INFO:teuthology.orchestra.run.smithi018.stdout: 5 198769 8.93 MB/sec execute 416 sec latency 2093.670 ms 2024-09-21T17:31:52.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:52 smithi018 ceph-mon[63924]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T17:31:52.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:52 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 202/1644 objects degraded (12.287%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:52.228 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:52 smithi018 ceph-mon[63924]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T17:31:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:52 smithi157 ceph-mon[49460]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T17:31:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:52 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 202/1644 objects degraded (12.287%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T17:31:52.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:52 smithi157 ceph-mon[49460]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T17:31:52.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 198994 8.93 MB/sec execute 417 sec latency 765.736 ms 2024-09-21T17:31:53.267 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:53 smithi018 ceph-mon[63924]: pgmap v174: 97 pgs: 6 remapped+peering, 5 active+undersized, 12 active+undersized+degraded, 74 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.4 MiB/s wr, 339 op/s; 67/1683 objects degraded (3.981%); 257 KiB/s, 17 objects/s recovering 2024-09-21T17:31:53.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:53 smithi018 ceph-mon[63924]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T17:31:53.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:53.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:53.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:31:53.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:31:53.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:53.268 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:53 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:53 smithi157 ceph-mon[49460]: pgmap v174: 97 pgs: 6 remapped+peering, 5 active+undersized, 12 active+undersized+degraded, 74 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.4 MiB/s wr, 339 op/s; 67/1683 objects degraded (3.981%); 257 KiB/s, 17 objects/s recovering 2024-09-21T17:31:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:53 smithi157 ceph-mon[49460]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T17:31:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:31:53.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:31:53.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:53.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:53 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:31:53.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 199323 8.92 MB/sec execute 418 sec latency 869.175 ms 2024-09-21T17:31:54.337 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:31:54.337 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:54.337 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:54.338 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:54.338 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T17:31:54.338 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:54 smithi018 ceph-mon[63924]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T17:31:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:31:54.600 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:54.600 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:54.600 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:31:54.600 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T17:31:54.600 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:54 smithi157 ceph-mon[49460]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T17:31:54.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 199660 8.91 MB/sec execute 419 sec latency 44.831 ms 2024-09-21T17:31:55.368 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:55 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T17:31:55.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:55 smithi018 ceph-mon[63924]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-21T17:31:55.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:55 smithi018 ceph-mon[63924]: pgmap v177: 97 pgs: 2 peering, 4 remapped+peering, 91 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 360 op/s; 748 KiB/s, 37 objects/s recovering 2024-09-21T17:31:55.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:55 smithi018 ceph-mon[63924]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/1683 objects degraded (3.981%), 12 pgs degraded) 2024-09-21T17:31:55.369 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:55 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:31:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:55 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T17:31:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:55 smithi157 ceph-mon[49460]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-21T17:31:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:55 smithi157 ceph-mon[49460]: pgmap v177: 97 pgs: 2 peering, 4 remapped+peering, 91 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 360 op/s; 748 KiB/s, 37 objects/s recovering 2024-09-21T17:31:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:55 smithi157 ceph-mon[49460]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/1683 objects degraded (3.981%), 12 pgs degraded) 2024-09-21T17:31:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:55 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:31:55.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 200052 8.92 MB/sec execute 420 sec latency 62.279 ms 2024-09-21T17:31:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 200650 8.93 MB/sec execute 421 sec latency 41.254 ms 2024-09-21T17:31:57.408 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:57 smithi018 ceph-mon[63924]: pgmap v178: 97 pgs: 2 peering, 95 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.6 MiB/s wr, 441 op/s; 1.7 MiB/s, 31 objects/s recovering 2024-09-21T17:31:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:57 smithi157 ceph-mon[49460]: pgmap v178: 97 pgs: 2 peering, 95 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.6 MiB/s wr, 441 op/s; 1.7 MiB/s, 31 objects/s recovering 2024-09-21T17:31:57.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 201037 8.95 MB/sec execute 422 sec latency 38.948 ms 2024-09-21T17:31:58.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 201348 8.96 MB/sec execute 423 sec latency 43.460 ms 2024-09-21T17:31:59.470 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:31:59 smithi018 ceph-mon[63924]: pgmap v179: 97 pgs: 2 peering, 95 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 2.9 MiB/s wr, 345 op/s; 1.2 MiB/s, 21 objects/s recovering 2024-09-21T17:31:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:31:59 smithi157 ceph-mon[49460]: pgmap v179: 97 pgs: 2 peering, 95 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 2.9 MiB/s wr, 345 op/s; 1.2 MiB/s, 21 objects/s recovering 2024-09-21T17:31:59.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 201656 8.95 MB/sec execute 424 sec latency 48.653 ms 2024-09-21T17:32:00.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 202063 8.95 MB/sec execute 425 sec latency 45.260 ms 2024-09-21T17:32:01.249 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:01 smithi018 ceph-mon[63924]: pgmap v180: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.8 MiB/s wr, 301 op/s; 949 KiB/s, 19 objects/s recovering 2024-09-21T17:32:01.250 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:01.250 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:32:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:01 smithi157 ceph-mon[49460]: pgmap v180: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.8 MiB/s wr, 301 op/s; 949 KiB/s, 19 objects/s recovering 2024-09-21T17:32:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:32:01.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 202382 8.95 MB/sec execute 426 sec latency 52.558 ms 2024-09-21T17:32:02.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 202750 8.95 MB/sec execute 427 sec latency 41.162 ms 2024-09-21T17:32:03.293 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:03 smithi018 ceph-mon[63924]: pgmap v181: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.0 MiB/s wr, 450 op/s; 807 KiB/s, 16 objects/s recovering 2024-09-21T17:32:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:03 smithi157 ceph-mon[49460]: pgmap v181: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.0 MiB/s wr, 450 op/s; 807 KiB/s, 16 objects/s recovering 2024-09-21T17:32:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 203151 8.94 MB/sec execute 428 sec latency 50.969 ms 2024-09-21T17:32:04.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 203448 8.93 MB/sec execute 429 sec latency 48.842 ms 2024-09-21T17:32:05.337 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:05 smithi018 ceph-mon[63924]: pgmap v182: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 503 op/s; 547 KiB/s, 7 objects/s recovering 2024-09-21T17:32:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:05 smithi157 ceph-mon[49460]: pgmap v182: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 503 op/s; 547 KiB/s, 7 objects/s recovering 2024-09-21T17:32:05.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 203811 8.94 MB/sec execute 430 sec latency 62.191 ms 2024-09-21T17:32:06.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 204297 8.95 MB/sec execute 431 sec latency 69.415 ms 2024-09-21T17:32:06.873 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:06.873 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:06 smithi018 ceph-mon[63924]: pgmap v183: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.8 MiB/s wr, 589 op/s; 475 KiB/s, 6 objects/s recovering 2024-09-21T17:32:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:06 smithi157 ceph-mon[49460]: pgmap v183: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.8 MiB/s wr, 589 op/s; 475 KiB/s, 6 objects/s recovering 2024-09-21T17:32:07.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 204616 8.97 MB/sec execute 432 sec latency 81.747 ms 2024-09-21T17:32:08.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 204920 8.97 MB/sec execute 433 sec latency 61.045 ms 2024-09-21T17:32:09.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:09 smithi018 ceph-mon[63924]: pgmap v184: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 429 op/s; 0 B/s, 1 objects/s recovering 2024-09-21T17:32:09.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:09 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T17:32:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:09 smithi157 ceph-mon[49460]: pgmap v184: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 429 op/s; 0 B/s, 1 objects/s recovering 2024-09-21T17:32:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:09 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T17:32:09.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 205185 8.96 MB/sec execute 434 sec latency 56.181 ms 2024-09-21T17:32:10.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:10 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T17:32:10.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:10 smithi018 ceph-mon[63924]: Upgrade: osd.4 is safe to restart 2024-09-21T17:32:10.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:10.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T17:32:10.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:32:10.452 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:10 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T17:32:10.452 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:10 smithi157 ceph-mon[49460]: Upgrade: osd.4 is safe to restart 2024-09-21T17:32:10.452 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:10.452 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T17:32:10.452 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:32:10.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 205509 8.96 MB/sec execute 435 sec latency 44.107 ms 2024-09-21T17:32:11.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:11 smithi018 ceph-mon[63924]: pgmap v185: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.2 MiB/s wr, 533 op/s; 0 B/s, 1 objects/s recovering 2024-09-21T17:32:11.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:11 smithi018 ceph-mon[63924]: Upgrade: Updating osd.4 2024-09-21T17:32:11.277 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:11 smithi018 ceph-mon[63924]: Deploying daemon osd.4 on smithi157 2024-09-21T17:32:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:11 smithi157 ceph-mon[49460]: pgmap v185: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.2 MiB/s wr, 533 op/s; 0 B/s, 1 objects/s recovering 2024-09-21T17:32:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:11 smithi157 ceph-mon[49460]: Upgrade: Updating osd.4 2024-09-21T17:32:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:11 smithi157 ceph-mon[49460]: Deploying daemon osd.4 on smithi157 2024-09-21T17:32:11.777 INFO:teuthology.orchestra.run.smithi018.stdout: 5 205700 8.95 MB/sec execute 436 sec latency 61.202 ms 2024-09-21T17:32:12.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 205882 8.95 MB/sec execute 437 sec latency 60.228 ms 2024-09-21T17:32:13.256 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:13 smithi157 ceph-mon[49460]: pgmap v186: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.5 MiB/s wr, 640 op/s 2024-09-21T17:32:13.325 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:13 smithi018 ceph-mon[63924]: pgmap v186: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.5 MiB/s wr, 640 op/s 2024-09-21T17:32:13.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 206085 8.93 MB/sec execute 438 sec latency 86.887 ms 2024-09-21T17:32:14.278 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:14 smithi157 systemd[1]: Stopping Ceph osd.4 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:32:14.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 206337 8.92 MB/sec execute 439 sec latency 75.089 ms 2024-09-21T17:32:14.778 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[30051]: 2024-09-21T17:32:14.388+0000 7f6c02abd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:32:14.778 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[30051]: 2024-09-21T17:32:14.388+0000 7f6c02abd700 -1 osd.4 116 *** Got signal Terminated *** 2024-09-21T17:32:14.779 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[30051]: 2024-09-21T17:32:14.388+0000 7f6c02abd700 -1 osd.4 116 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:32:15.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:15 smithi018 ceph-mon[63924]: pgmap v187: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.8 MiB/s wr, 499 op/s 2024-09-21T17:32:15.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:15 smithi018 ceph-mon[63924]: osd.4 marked itself down and dead 2024-09-21T17:32:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:15 smithi157 ceph-mon[49460]: pgmap v187: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.8 MiB/s wr, 499 op/s 2024-09-21T17:32:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:15 smithi157 ceph-mon[49460]: osd.4 marked itself down and dead 2024-09-21T17:32:15.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 206406 8.90 MB/sec execute 440 sec latency 710.961 ms 2024-09-21T17:32:16.028 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:15 smithi157 podman[52878]: 2024-09-21 17:32:15.750213605 +0000 UTC m=+1.515704879 container died 894457f33d1bcd1e3b0c4a7f76bcea86791e1100a7a57cf6ba3e9f62fda2cc5c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, ceph=True, version=8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, vcs-type=git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-21T17:32:16.378 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:16 smithi018 ceph-mon[63924]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:32:16.378 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:16 smithi018 ceph-mon[63924]: osdmap e117: 8 total, 7 up, 8 in 2024-09-21T17:32:16.378 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:16.379 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:32:16.382 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:16 smithi157 ceph-mon[49460]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:32:16.382 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:16 smithi157 ceph-mon[49460]: osdmap e117: 8 total, 7 up, 8 in 2024-09-21T17:32:16.383 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:16.383 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:32:16.383 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:16 smithi157 podman[52878]: 2024-09-21 17:32:16.162535567 +0000 UTC m=+1.928026848 container cleanup 894457f33d1bcd1e3b0c4a7f76bcea86791e1100a7a57cf6ba3e9f62fda2cc5c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, release=754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, ceph=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vcs-type=git, io.openshift.expose-services=, vendor=Red Hat, Inc., distribution-scope=public, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD) 2024-09-21T17:32:16.383 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:16 smithi157 bash[52878]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4 2024-09-21T17:32:16.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 206571 8.89 MB/sec execute 441 sec latency 1138.871 ms 2024-09-21T17:32:16.778 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:16 smithi157 podman[52893]: 2024-09-21 17:32:16.380962836 +0000 UTC m=+0.626449136 container remove 894457f33d1bcd1e3b0c4a7f76bcea86791e1100a7a57cf6ba3e9f62fda2cc5c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, maintainer=Guillaume Abrioux , name=centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, vcs-type=git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T17:32:17.133 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:16 smithi157 podman[52979]: 2024-09-21 17:32:16.754882688 +0000 UTC m=+0.018681222 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:32:17.133 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:16 smithi157 podman[52979]: 2024-09-21 17:32:16.881878779 +0000 UTC m=+0.145677298 container create b355de655823f57260bb80ae422d7515e80e9467658c7fc4f1011dd9d791ffab (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:32:17.156 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:17 smithi018 ceph-mon[63924]: pgmap v189: 97 pgs: 26 peering, 7 stale+active+clean, 64 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.2 MiB/s wr, 427 op/s 2024-09-21T17:32:17.156 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:17 smithi018 ceph-mon[63924]: osdmap e118: 8 total, 7 up, 8 in 2024-09-21T17:32:17.156 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:17 smithi018 ceph-mon[63924]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-21T17:32:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:17 smithi157 ceph-mon[49460]: pgmap v189: 97 pgs: 26 peering, 7 stale+active+clean, 64 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.2 MiB/s wr, 427 op/s 2024-09-21T17:32:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:17 smithi157 ceph-mon[49460]: osdmap e118: 8 total, 7 up, 8 in 2024-09-21T17:32:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:17 smithi157 ceph-mon[49460]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-21T17:32:17.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:17 smithi157 podman[52979]: 2024-09-21 17:32:17.152601661 +0000 UTC m=+0.416400177 container init b355de655823f57260bb80ae422d7515e80e9467658c7fc4f1011dd9d791ffab (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:32:17.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:17 smithi157 podman[52979]: 2024-09-21 17:32:17.156197444 +0000 UTC m=+0.419995968 container start b355de655823f57260bb80ae422d7515e80e9467658c7fc4f1011dd9d791ffab (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T17:32:17.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:17 smithi157 podman[52979]: 2024-09-21 17:32:17.269577558 +0000 UTC m=+0.533376085 container attach b355de655823f57260bb80ae422d7515e80e9467658c7fc4f1011dd9d791ffab (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T17:32:17.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:17 smithi157 podman[52979]: 2024-09-21 17:32:17.365460686 +0000 UTC m=+0.629259213 container died b355de655823f57260bb80ae422d7515e80e9467658c7fc4f1011dd9d791ffab (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T17:32:17.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 206894 8.88 MB/sec execute 442 sec latency 48.356 ms 2024-09-21T17:32:17.836 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:32:18.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:18 smithi157 ceph-mon[49460]: Health check failed: Degraded data redundancy: 29/1584 objects degraded (1.831%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:18.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 podman[53002]: 2024-09-21 17:32:18.049697414 +0000 UTC m=+0.678705157 container remove b355de655823f57260bb80ae422d7515e80e9467658c7fc4f1011dd9d791ffab (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-21T17:32:18.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-21T17:32:18.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 systemd[1]: Stopped Ceph osd.4 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:32:18.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.4.service: Consumed 3min 33.748s CPU time. 2024-09-21T17:32:18.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:18 smithi018 ceph-mon[63924]: Health check failed: Degraded data redundancy: 29/1584 objects degraded (1.831%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:18.680 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 systemd[1]: Starting Ceph osd.4 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:32:18.680 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 podman[53102]: 2024-09-21 17:32:18.407937241 +0000 UTC m=+0.015742243 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:32:18.680 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 podman[53102]: 2024-09-21 17:32:18.534367131 +0000 UTC m=+0.142172116 container create 84d0cf7bed9ad6a6858874384fb3fe3e2d46746a4e2d92d04833a07c236d603c (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:32:18.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 207202 8.89 MB/sec execute 443 sec latency 41.613 ms 2024-09-21T17:32:18.945 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 podman[53102]: 2024-09-21 17:32:18.804075117 +0000 UTC m=+0.411880105 container init 84d0cf7bed9ad6a6858874384fb3fe3e2d46746a4e2d92d04833a07c236d603c (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T17:32:18.945 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 podman[53102]: 2024-09-21 17:32:18.807637392 +0000 UTC m=+0.415442385 container start 84d0cf7bed9ad6a6858874384fb3fe3e2d46746a4e2d92d04833a07c236d603c (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-21T17:32:18.945 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 podman[53102]: 2024-09-21 17:32:18.912030658 +0000 UTC m=+0.519835653 container attach 84d0cf7bed9ad6a6858874384fb3fe3e2d46746a4e2d92d04833a07c236d603c (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T17:32:18.965 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:32:18.965 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (6m) 33s ago 16m 19.7M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:32:18.965 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (3m) 33s ago 16m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:32:18.965 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (3m) 3m ago 15m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:32:18.965 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (6m) 33s ago 15m 47.5M - dad864ee21e9 48d78bd31599 2024-09-21T17:32:18.965 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (9m) 33s ago 9m 38.4M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (9m) 3m ago 9m 15.7M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (6m) 33s ago 17m 520M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (5m) 3m ago 15m 455M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (4m) 33s ago 17m 51.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (4m) 3m ago 15m 35.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (9m) 33s ago 9m 276M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (9m) 3m ago 9m 86.8M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (6m) 33s ago 15m 10.1M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (6m) 3m ago 15m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (3m) 33s ago 14m 337M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (2m) 33s ago 14m 272M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:32:18.966 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (90s) 33s ago 13m 218M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:32:18.967 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (41s) 33s ago 13m 13.3M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:32:18.967 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (12m) 3m ago 12m 483M 2704M 17.2.0 e1d6a67b021e 894457f33d1b 2024-09-21T17:32:18.967 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (12m) 3m ago 12m 536M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:32:18.967 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (11m) 3m ago 11m 456M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:32:18.967 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (11m) 3m ago 11m 328M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:32:18.967 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (5m) 33s ago 15m 53.1M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:32:19.200 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:32:19.201 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 bash[53102]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:32:19.201 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:32:19.201 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:18 smithi157 bash[53102]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:32:19.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:19 smithi018 ceph-mon[63924]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:19.213 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:19 smithi018 ceph-mon[63924]: pgmap v191: 97 pgs: 3 active+undersized, 26 peering, 5 stale+active+clean, 4 active+undersized+degraded, 59 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 361 op/s; 29/1584 objects degraded (1.831%) 2024-09-21T17:32:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:19 smithi157 ceph-mon[49460]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:19 smithi157 ceph-mon[49460]: pgmap v191: 97 pgs: 3 active+undersized, 26 peering, 5 stale+active+clean, 4 active+undersized+degraded, 59 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 361 op/s; 29/1584 objects degraded (1.831%) 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 4 2024-09-21T17:32:19.743 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:32:19.744 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:32:19.744 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:32:19.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:32:19.744 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:32:19.744 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T17:32:19.744 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 8 2024-09-21T17:32:19.744 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:32:19.744 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:32:19.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 207637 8.90 MB/sec execute 444 sec latency 89.731 ms 2024-09-21T17:32:20.227 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:20 smithi018 ceph-mon[63924]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:20.227 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:20 smithi018 ceph-mon[63924]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:20.227 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:20 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1236062863' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:20.331 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:32:20.331 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:32:20.331 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:32:20.331 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:32:20.331 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:32:20.332 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:32:20.332 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:32:20.332 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:32:20.332 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:32:20.332 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "10/23 daemons upgraded", 2024-09-21T17:32:20.332 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:32:20.332 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:32:20.332 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:32:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-mon[49460]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-mon[49460]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1236062863' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:20.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T17:32:20.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 bash[53102]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T17:32:20.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T17:32:20.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 bash[53102]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T17:32:20.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 208024 8.90 MB/sec execute 445 sec latency 34.319 ms 2024-09-21T17:32:21.012 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 osds down; Reduced data availability: 4 pgs inactive, 5 pgs peering; Degraded data redundancy: 61/1545 objects degraded (3.948%), 7 pgs degraded 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: osd.4 (root=default,host=smithi157) is down 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] PG_AVAILABILITY: Reduced data availability: 4 pgs inactive, 5 pgs peering 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.f is stuck peering for 2m, current state peering, last acting [0,7] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.1 is stuck peering for 9m, current state peering, last acting [5,6] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.e is stuck peering for 2m, current state peering, last acting [6,1] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.13 is stuck peering for 2m, current state peering, last acting [6,1] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 4.18 is stuck peering for 81s, current state peering, last acting [6,2] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 61/1545 objects degraded (3.948%), 7 pgs degraded 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.5 is active+undersized+degraded, acting [2,0] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-21T17:32:21.013 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-21T17:32:21.014 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-21T17:32:21.014 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-21T17:32:21.028 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T17:32:21.028 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 bash[53102]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T17:32:21.029 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T17:32:21.029 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 bash[53102]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T17:32:21.029 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T17:32:21.029 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 bash[53102]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T17:32:21.029 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T17:32:21.029 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 bash[53102]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T17:32:21.029 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate[53116]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T17:32:21.030 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 bash[53102]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T17:32:21.030 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:20 smithi157 podman[53102]: 2024-09-21 17:32:20.64408106 +0000 UTC m=+2.251886048 container died 84d0cf7bed9ad6a6858874384fb3fe3e2d46746a4e2d92d04833a07c236d603c (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:32:21.238 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:21 smithi018 ceph-mon[63924]: pgmap v192: 97 pgs: 8 active+undersized, 26 peering, 7 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 1.8 MiB/s wr, 197 op/s; 61/1545 objects degraded (3.948%) 2024-09-21T17:32:21.238 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:21 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3497711375' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:32:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:21 smithi157 ceph-mon[49460]: pgmap v192: 97 pgs: 8 active+undersized, 26 peering, 7 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 1.8 MiB/s wr, 197 op/s; 61/1545 objects degraded (3.948%) 2024-09-21T17:32:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:21 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3497711375' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:32:21.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 208325 8.91 MB/sec execute 446 sec latency 37.550 ms 2024-09-21T17:32:21.830 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:21 smithi157 podman[53259]: 2024-09-21 17:32:21.552168036 +0000 UTC m=+0.897401199 container remove 84d0cf7bed9ad6a6858874384fb3fe3e2d46746a4e2d92d04833a07c236d603c (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T17:32:21.830 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:21 smithi157 podman[53332]: 2024-09-21 17:32:21.680979931 +0000 UTC m=+0.015728865 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:32:21.830 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:21 smithi157 podman[53332]: 2024-09-21 17:32:21.828531749 +0000 UTC m=+0.163280669 container create 2552dce6837e1a43e8deeb373f2c8a23e56d667d3825970144cdc09fe821ad94 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T17:32:22.262 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:22 smithi018 ceph-mon[63924]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:22.528 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:22 smithi157 podman[53332]: 2024-09-21 17:32:22.081039357 +0000 UTC m=+0.415788278 container init 2552dce6837e1a43e8deeb373f2c8a23e56d667d3825970144cdc09fe821ad94 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T17:32:22.528 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:22 smithi157 podman[53332]: 2024-09-21 17:32:22.084443717 +0000 UTC m=+0.419192638 container start 2552dce6837e1a43e8deeb373f2c8a23e56d667d3825970144cdc09fe821ad94 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T17:32:22.528 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:22 smithi157 ceph-osd[53350]: -- 172.21.15.157:0/3035075355 <== mon.1 v2:172.21.15.157:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5597d37581e0 con 0x5597d36e4000 2024-09-21T17:32:22.528 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:22 smithi157 bash[53332]: 2552dce6837e1a43e8deeb373f2c8a23e56d667d3825970144cdc09fe821ad94 2024-09-21T17:32:22.528 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:22 smithi157 systemd[1]: Started Ceph osd.4 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:32:22.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:22 smithi157 ceph-mon[49460]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:22.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 208568 8.91 MB/sec execute 447 sec latency 52.195 ms 2024-09-21T17:32:23.292 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:23 smithi018 ceph-mon[63924]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.1 MiB/s wr, 455 op/s; 261/1656 objects degraded (15.761%) 2024-09-21T17:32:23.292 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:23 smithi018 ceph-mon[63924]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-21T17:32:23.292 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:23.293 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:23.293 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:23 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:32:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:23 smithi157 ceph-mon[49460]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.1 MiB/s wr, 455 op/s; 261/1656 objects degraded (15.761%) 2024-09-21T17:32:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:23 smithi157 ceph-mon[49460]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-21T17:32:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:23 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:32:23.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 208885 8.91 MB/sec execute 448 sec latency 40.849 ms 2024-09-21T17:32:23.870 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:23 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:32:23.715+0000 7efdbec92640 -1 Falling back to public interface 2024-09-21T17:32:24.315 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:24 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 267/1704 objects degraded (15.669%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:24 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 267/1704 objects degraded (15.669%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:24.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 209140 8.90 MB/sec execute 449 sec latency 43.856 ms 2024-09-21T17:32:25.342 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:25 smithi018 ceph-mon[63924]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 499 op/s; 267/1704 objects degraded (15.669%) 2024-09-21T17:32:25.706 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:25 smithi157 ceph-mon[49460]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 499 op/s; 267/1704 objects degraded (15.669%) 2024-09-21T17:32:25.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 209300 8.89 MB/sec execute 450 sec latency 72.724 ms 2024-09-21T17:32:26.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 209519 8.88 MB/sec execute 451 sec latency 60.879 ms 2024-09-21T17:32:27.397 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:27 smithi018 ceph-mon[63924]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.3 MiB/s wr, 403 op/s; 273/1698 objects degraded (16.078%) 2024-09-21T17:32:27.397 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:27.398 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:27.398 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:27.398 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:27.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:27 smithi157 ceph-mon[49460]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.3 MiB/s wr, 403 op/s; 273/1698 objects degraded (16.078%) 2024-09-21T17:32:27.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:27.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:27.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:27.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:27.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 209813 8.88 MB/sec execute 452 sec latency 83.539 ms 2024-09-21T17:32:28.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 210102 8.86 MB/sec execute 453 sec latency 48.684 ms 2024-09-21T17:32:29.458 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:29 smithi018 ceph-mon[63924]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.8 MiB/s wr, 346 op/s; 275/1701 objects degraded (16.167%) 2024-09-21T17:32:29.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:29 smithi157 ceph-mon[49460]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.8 MiB/s wr, 346 op/s; 275/1701 objects degraded (16.167%) 2024-09-21T17:32:29.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 210385 8.85 MB/sec execute 454 sec latency 54.282 ms 2024-09-21T17:32:30.028 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:29 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:32:29.732+0000 7efdbec92640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-21T17:32:30.501 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:30 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 278/1713 objects degraded (16.229%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:30 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 278/1713 objects degraded (16.229%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:30.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 210598 8.85 MB/sec execute 455 sec latency 64.714 ms 2024-09-21T17:32:31.278 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:32:31.014+0000 7efdbec92640 -1 osd.4 116 log_to_monitors true 2024-09-21T17:32:31.525 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 376 op/s; 278/1713 objects degraded (16.229%) 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='osd.4 [v2:172.21.15.157:6800/4176698138,v1:172.21.15.157:6801/4176698138]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T17:32:31.526 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:32:31.527 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:31.527 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:31.527 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:31.527 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T17:32:31.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 376 op/s; 278/1713 objects degraded (16.229%) 2024-09-21T17:32:31.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='osd.4 [v2:172.21.15.157:6800/4176698138,v1:172.21.15.157:6801/4176698138]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:31.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:31.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T17:32:31.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 210918 8.86 MB/sec execute 456 sec latency 64.691 ms 2024-09-21T17:32:32.278 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:32:31 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:32:31.989+0000 7efdb2a34640 -1 osd.4 116 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:32:32.329 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:32 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T17:32:32.329 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:32 smithi018 ceph-mon[63924]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-21T17:32:32.329 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:32 smithi018 ceph-mon[63924]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T17:32:32.329 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:32 smithi018 ceph-mon[63924]: osdmap e119: 8 total, 7 up, 8 in 2024-09-21T17:32:32.329 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:32 smithi018 ceph-mon[63924]: from='osd.4 [v2:172.21.15.157:6800/4176698138,v1:172.21.15.157:6801/4176698138]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:32:32.330 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:32 smithi018 ceph-mon[63924]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:32:32.330 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:32 smithi018 ceph-mon[63924]: from='osd.4 ' entity='osd.4' 2024-09-21T17:32:32.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 211300 8.86 MB/sec execute 457 sec latency 72.762 ms 2024-09-21T17:32:32.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:32 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T17:32:32.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:32 smithi157 ceph-mon[49460]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-21T17:32:32.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:32 smithi157 ceph-mon[49460]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T17:32:32.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:32 smithi157 ceph-mon[49460]: osdmap e119: 8 total, 7 up, 8 in 2024-09-21T17:32:32.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:32 smithi157 ceph-mon[49460]: from='osd.4 [v2:172.21.15.157:6800/4176698138,v1:172.21.15.157:6801/4176698138]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:32:32.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:32 smithi157 ceph-mon[49460]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:32:32.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:32 smithi157 ceph-mon[49460]: from='osd.4 ' entity='osd.4' 2024-09-21T17:32:33.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:33 smithi018 ceph-mon[63924]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 398 op/s; 243/1572 objects degraded (15.458%) 2024-09-21T17:32:33.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:33 smithi018 ceph-mon[63924]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:32:33.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:33 smithi018 ceph-mon[63924]: osd.4 [v2:172.21.15.157:6800/4176698138,v1:172.21.15.157:6801/4176698138] boot 2024-09-21T17:32:33.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:33 smithi018 ceph-mon[63924]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T17:32:33.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:33 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T17:32:33.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:33 smithi157 ceph-mon[49460]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 398 op/s; 243/1572 objects degraded (15.458%) 2024-09-21T17:32:33.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:33 smithi157 ceph-mon[49460]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:32:33.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:33 smithi157 ceph-mon[49460]: osd.4 [v2:172.21.15.157:6800/4176698138,v1:172.21.15.157:6801/4176698138] boot 2024-09-21T17:32:33.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:33 smithi157 ceph-mon[49460]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T17:32:33.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:33 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T17:32:33.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 211394 8.84 MB/sec execute 458 sec latency 795.482 ms 2024-09-21T17:32:34.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 211458 8.83 MB/sec execute 459 sec latency 1795.530 ms 2024-09-21T17:32:35.151 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:34 smithi018 ceph-mon[63924]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T17:32:35.151 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:34 smithi018 ceph-mon[63924]: pgmap v202: 97 pgs: 3 peering, 12 active+undersized, 26 active+undersized+degraded, 56 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.3 MiB/s wr, 414 op/s; 216/1557 objects degraded (13.873%) 2024-09-21T17:32:35.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:35 smithi157 ceph-mon[49460]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T17:32:35.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:35 smithi157 ceph-mon[49460]: pgmap v202: 97 pgs: 3 peering, 12 active+undersized, 26 active+undersized+degraded, 56 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.3 MiB/s wr, 414 op/s; 216/1557 objects degraded (13.873%) 2024-09-21T17:32:35.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 211746 8.84 MB/sec execute 460 sec latency 2061.227 ms 2024-09-21T17:32:36.029 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:35 smithi018 ceph-mon[63924]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T17:32:36.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:35 smithi157 ceph-mon[49460]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T17:32:36.778 INFO:teuthology.orchestra.run.smithi018.stdout: 5 211946 8.84 MB/sec execute 461 sec latency 753.884 ms 2024-09-21T17:32:37.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:37 smithi018 ceph-mon[63924]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T17:32:37.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:37 smithi018 ceph-mon[63924]: pgmap v205: 97 pgs: 8 peering, 11 active+undersized, 8 active+undersized+degraded, 70 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.7 MiB/s wr, 373 op/s; 68/1596 objects degraded (4.261%); 345 KiB/s, 53 objects/s recovering 2024-09-21T17:32:37.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:37 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 216/1557 objects degraded (13.873%), 26 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:37.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:37 smithi157 ceph-mon[49460]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T17:32:37.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:37 smithi157 ceph-mon[49460]: pgmap v205: 97 pgs: 8 peering, 11 active+undersized, 8 active+undersized+degraded, 70 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.7 MiB/s wr, 373 op/s; 68/1596 objects degraded (4.261%); 345 KiB/s, 53 objects/s recovering 2024-09-21T17:32:37.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:37 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 216/1557 objects degraded (13.873%), 26 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:37.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 212140 8.83 MB/sec execute 462 sec latency 1028.537 ms 2024-09-21T17:32:38.031 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:38 smithi018 ceph-mon[63924]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T17:32:38.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:38 smithi157 ceph-mon[49460]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T17:32:38.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 212441 8.83 MB/sec execute 463 sec latency 48.399 ms 2024-09-21T17:32:39.055 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:39 smithi018 ceph-mon[63924]: pgmap v207: 97 pgs: 10 peering, 87 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 355 op/s; 1.5 MiB/s, 53 keys/s, 74 objects/s recovering 2024-09-21T17:32:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:39 smithi157 ceph-mon[49460]: pgmap v207: 97 pgs: 10 peering, 87 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 MiB/s wr, 355 op/s; 1.5 MiB/s, 53 keys/s, 74 objects/s recovering 2024-09-21T17:32:39.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 212748 8.82 MB/sec execute 464 sec latency 50.780 ms 2024-09-21T17:32:40.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:40 smithi018 ceph-mon[63924]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 68/1596 objects degraded (4.261%), 8 pgs degraded) 2024-09-21T17:32:40.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:40 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:32:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:40 smithi157 ceph-mon[49460]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 68/1596 objects degraded (4.261%), 8 pgs degraded) 2024-09-21T17:32:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:40 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:32:40.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 212989 8.81 MB/sec execute 465 sec latency 102.217 ms 2024-09-21T17:32:41.283 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:41 smithi018 ceph-mon[63924]: pgmap v208: 97 pgs: 7 peering, 90 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 1.1 MiB/s, 36 keys/s, 54 objects/s recovering 2024-09-21T17:32:41.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:41 smithi157 ceph-mon[49460]: pgmap v208: 97 pgs: 7 peering, 90 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 1.1 MiB/s, 36 keys/s, 54 objects/s recovering 2024-09-21T17:32:41.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 213389 8.82 MB/sec execute 466 sec latency 52.895 ms 2024-09-21T17:32:42.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 213826 8.81 MB/sec execute 467 sec latency 67.016 ms 2024-09-21T17:32:42.813 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:42 smithi018 ceph-mon[63924]: pgmap v209: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.3 MiB/s wr, 494 op/s; 952 KiB/s, 30 keys/s, 46 objects/s recovering 2024-09-21T17:32:43.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:42 smithi157 ceph-mon[49460]: pgmap v209: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.3 MiB/s wr, 494 op/s; 952 KiB/s, 30 keys/s, 46 objects/s recovering 2024-09-21T17:32:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 214202 8.81 MB/sec execute 468 sec latency 42.293 ms 2024-09-21T17:32:44.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 214592 8.81 MB/sec execute 469 sec latency 94.295 ms 2024-09-21T17:32:45.119 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:45 smithi018 ceph-mon[63924]: pgmap v210: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 541 op/s; 835 KiB/s, 26 keys/s, 40 objects/s recovering 2024-09-21T17:32:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:45 smithi157 ceph-mon[49460]: pgmap v210: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 541 op/s; 835 KiB/s, 26 keys/s, 40 objects/s recovering 2024-09-21T17:32:45.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 215024 8.82 MB/sec execute 470 sec latency 97.217 ms 2024-09-21T17:32:46.679 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:46.679 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:32:46.679 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:46 smithi018 ceph-mon[63924]: pgmap v211: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 408 op/s; 534 KiB/s, 21 keys/s, 11 objects/s recovering 2024-09-21T17:32:46.680 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T17:32:46.680 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:46 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T17:32:46.680 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:46 smithi018 ceph-mon[63924]: Upgrade: osd.5 is safe to restart 2024-09-21T17:32:46.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 215386 8.84 MB/sec execute 471 sec latency 108.215 ms 2024-09-21T17:32:46.868 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:46.868 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:32:46.868 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:46 smithi157 ceph-mon[49460]: pgmap v211: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 408 op/s; 534 KiB/s, 21 keys/s, 11 objects/s recovering 2024-09-21T17:32:46.868 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T17:32:46.868 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:46 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T17:32:46.868 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:46 smithi157 ceph-mon[49460]: Upgrade: osd.5 is safe to restart 2024-09-21T17:32:47.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 215691 8.84 MB/sec execute 472 sec latency 44.356 ms 2024-09-21T17:32:48.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 216161 8.85 MB/sec execute 473 sec latency 48.397 ms 2024-09-21T17:32:49.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:49 smithi018 ceph-mon[63924]: Upgrade: Updating osd.5 2024-09-21T17:32:49.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:49 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:49.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:49 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T17:32:49.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:49 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:32:49.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:49 smithi018 ceph-mon[63924]: pgmap v212: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 377 op/s; 33 KiB/s, 2 objects/s recovering 2024-09-21T17:32:49.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:49 smithi018 ceph-mon[63924]: Deploying daemon osd.5 on smithi157 2024-09-21T17:32:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:49 smithi157 ceph-mon[49460]: Upgrade: Updating osd.5 2024-09-21T17:32:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:49 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:49 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T17:32:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:49 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:32:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:49 smithi157 ceph-mon[49460]: pgmap v212: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 377 op/s; 33 KiB/s, 2 objects/s recovering 2024-09-21T17:32:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:49 smithi157 ceph-mon[49460]: Deploying daemon osd.5 on smithi157 2024-09-21T17:32:49.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 216528 8.85 MB/sec execute 474 sec latency 40.050 ms 2024-09-21T17:32:50.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 216829 8.85 MB/sec execute 475 sec latency 49.909 ms 2024-09-21T17:32:51.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:51 smithi018 ceph-mon[63924]: pgmap v213: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.4 MiB/s wr, 415 op/s; 30 KiB/s, 1 objects/s recovering 2024-09-21T17:32:51.233 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:51 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:51.430 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:51 smithi157 ceph-mon[49460]: pgmap v213: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.4 MiB/s wr, 415 op/s; 30 KiB/s, 1 objects/s recovering 2024-09-21T17:32:51.430 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:51 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:32:51.562 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:32:51.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 217070 8.84 MB/sec execute 476 sec latency 98.798 ms 2024-09-21T17:32:52.170 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:51 smithi157 systemd[1]: Stopping Ceph osd.5 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:32:52.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[32997]: 2024-09-21T17:32:52.166+0000 7f14b0042700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:32:52.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[32997]: 2024-09-21T17:32:52.166+0000 7f14b0042700 -1 osd.5 124 *** Got signal Terminated *** 2024-09-21T17:32:52.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[32997]: 2024-09-21T17:32:52.166+0000 7f14b0042700 -1 osd.5 124 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:32:52.718 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:32:52.718 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (7m) 67s ago 16m 19.7M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:32:52.718 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (4m) 67s ago 16m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:32:52.718 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (4m) 26s ago 16m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:32:52.718 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (7m) 67s ago 16m 47.5M - dad864ee21e9 48d78bd31599 2024-09-21T17:32:52.718 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (10m) 67s ago 10m 38.4M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:32:52.718 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (10m) 26s ago 10m 16.4M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:32:52.718 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (6m) 67s ago 17m 520M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (5m) 26s ago 16m 458M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (5m) 67s ago 18m 51.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (5m) 26s ago 15m 42.0M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (10m) 67s ago 10m 276M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (10m) 26s ago 10m 87.3M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (7m) 67s ago 16m 10.1M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (6m) 26s ago 15m 10.9M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (3m) 67s ago 15m 337M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (2m) 67s ago 14m 272M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (2m) 67s ago 14m 218M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (75s) 67s ago 13m 13.3M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (31s) 26s ago 13m 11.1M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 2552dce6837e 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (12m) 26s ago 12m 489M 2704M 17.2.0 e1d6a67b021e f3c44c921e1e 2024-09-21T17:32:52.719 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (12m) 26s ago 12m 395M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:32:52.720 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (12m) 26s ago 12m 626M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:32:52.720 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (5m) 67s ago 15m 53.1M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:32:52.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 217478 8.83 MB/sec execute 477 sec latency 69.882 ms 2024-09-21T17:32:53.285 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:32:53.285 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:32:53.285 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:32:53.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:32:53.285 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:32:53.285 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:32:53.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 5 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 9 2024-09-21T17:32:53.286 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:32:53.287 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:32:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:53 smithi018 ceph-mon[63924]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:53 smithi018 ceph-mon[63924]: pgmap v214: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.0 MiB/s wr, 613 op/s 2024-09-21T17:32:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:53 smithi018 ceph-mon[63924]: osd.5 marked itself down and dead 2024-09-21T17:32:53.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:53 smithi018 ceph-mon[63924]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:53 smithi157 ceph-mon[49460]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:53 smithi157 ceph-mon[49460]: pgmap v214: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.0 MiB/s wr, 613 op/s 2024-09-21T17:32:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:53 smithi157 ceph-mon[49460]: osd.5 marked itself down and dead 2024-09-21T17:32:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:53 smithi157 ceph-mon[49460]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "11/23 daemons upgraded", 2024-09-21T17:32:53.657 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:32:53.658 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:32:53.658 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:32:53.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 217605 8.82 MB/sec execute 478 sec latency 657.287 ms 2024-09-21T17:32:54.028 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:53 smithi157 podman[54930]: 2024-09-21 17:32:53.745488468 +0000 UTC m=+1.704376505 container died f3c44c921e1e8d4f410f54cf8df74a416551a82e85387de586309613e8ddddf0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, ceph=True, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, version=8, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_BRANCH=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, release=754) 2024-09-21T17:32:54.093 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 osds down 2024-09-21T17:32:54.093 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T17:32:54.093 INFO:teuthology.orchestra.run.smithi018.stdout: osd.5 (root=default,host=smithi157) is down 2024-09-21T17:32:54.342 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:54 smithi018 ceph-mon[63924]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:54.342 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:54 smithi018 ceph-mon[63924]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:32:54.342 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:54 smithi018 ceph-mon[63924]: osdmap e125: 8 total, 7 up, 8 in 2024-09-21T17:32:54.342 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:54 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/808091883' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:54 smithi157 ceph-mon[49460]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:54 smithi157 ceph-mon[49460]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:32:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:54 smithi157 ceph-mon[49460]: osdmap e125: 8 total, 7 up, 8 in 2024-09-21T17:32:54.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:54 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/808091883' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:32:54.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:54 smithi157 podman[54930]: 2024-09-21 17:32:54.292532195 +0000 UTC m=+2.251420236 container cleanup f3c44c921e1e8d4f410f54cf8df74a416551a82e85387de586309613e8ddddf0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=centos-stream, version=8, ceph=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_CLEAN=True, release=754) 2024-09-21T17:32:54.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:54 smithi157 bash[54930]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5 2024-09-21T17:32:54.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 217714 8.80 MB/sec execute 479 sec latency 1061.571 ms 2024-09-21T17:32:55.009 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:54 smithi157 podman[54947]: 2024-09-21 17:32:54.636273429 +0000 UTC m=+0.888485248 container remove f3c44c921e1e8d4f410f54cf8df74a416551a82e85387de586309613e8ddddf0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64) 2024-09-21T17:32:55.117 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:55 smithi018 ceph-mon[63924]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:55.118 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:55 smithi018 ceph-mon[63924]: pgmap v216: 97 pgs: 7 peering, 9 stale+active+clean, 81 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 475 op/s 2024-09-21T17:32:55.118 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:55 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2388104036' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:32:55.118 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:55 smithi018 ceph-mon[63924]: osdmap e126: 8 total, 7 up, 8 in 2024-09-21T17:32:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:55 smithi157 ceph-mon[49460]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:32:55.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:55 smithi157 ceph-mon[49460]: pgmap v216: 97 pgs: 7 peering, 9 stale+active+clean, 81 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 475 op/s 2024-09-21T17:32:55.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:55 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2388104036' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:32:55.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:55 smithi157 ceph-mon[49460]: osdmap e126: 8 total, 7 up, 8 in 2024-09-21T17:32:55.279 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:55 smithi157 podman[55032]: 2024-09-21 17:32:54.910112805 +0000 UTC m=+0.017719055 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:32:55.279 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:55 smithi157 podman[55032]: 2024-09-21 17:32:55.045536942 +0000 UTC m=+0.153143176 container create ad9680a6a2ba2ba682fdc2b649266876b0518fc8032e784936bd88ebd2746fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:32:55.779 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:55 smithi157 podman[55032]: 2024-09-21 17:32:55.320195307 +0000 UTC m=+0.427801549 container init ad9680a6a2ba2ba682fdc2b649266876b0518fc8032e784936bd88ebd2746fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T17:32:55.779 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:55 smithi157 podman[55032]: 2024-09-21 17:32:55.324933906 +0000 UTC m=+0.432540147 container start ad9680a6a2ba2ba682fdc2b649266876b0518fc8032e784936bd88ebd2746fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:32:55.779 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:55 smithi157 podman[55032]: 2024-09-21 17:32:55.423645357 +0000 UTC m=+0.531251592 container attach ad9680a6a2ba2ba682fdc2b649266876b0518fc8032e784936bd88ebd2746fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T17:32:55.779 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:55 smithi157 podman[55032]: 2024-09-21 17:32:55.549461772 +0000 UTC m=+0.657068023 container died ad9680a6a2ba2ba682fdc2b649266876b0518fc8032e784936bd88ebd2746fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux ) 2024-09-21T17:32:55.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 217940 8.80 MB/sec execute 480 sec latency 62.583 ms 2024-09-21T17:32:56.158 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:56 smithi018 ceph-mon[63924]: Health check failed: Degraded data redundancy: 84/1566 objects degraded (5.364%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:56.347 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:56 smithi157 ceph-mon[49460]: Health check failed: Degraded data redundancy: 84/1566 objects degraded (5.364%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T17:32:56.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:56 smithi157 podman[55055]: 2024-09-21 17:32:56.34590619 +0000 UTC m=+0.788709556 container remove ad9680a6a2ba2ba682fdc2b649266876b0518fc8032e784936bd88ebd2746fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:32:56.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:56 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-21T17:32:56.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:56 smithi157 systemd[1]: Stopped Ceph osd.5 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:32:56.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:56 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.5.service: Consumed 4min 13.269s CPU time. 2024-09-21T17:32:56.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:56 smithi157 systemd[1]: Starting Ceph osd.5 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:32:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 218282 8.81 MB/sec execute 481 sec latency 48.132 ms 2024-09-21T17:32:57.127 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:56 smithi157 podman[55153]: 2024-09-21 17:32:56.712622478 +0000 UTC m=+0.019126736 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:32:57.127 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:56 smithi157 podman[55153]: 2024-09-21 17:32:56.84711203 +0000 UTC m=+0.153616284 container create 8b9845976b0b628fd283ddd2f7269f2f9fd668d4eedaa60e95f18efd56b33f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T17:32:57.175 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:57 smithi018 ceph-mon[63924]: pgmap v218: 97 pgs: 5 active+undersized, 7 peering, 4 stale+active+clean, 10 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 611 op/s; 84/1566 objects degraded (5.364%) 2024-09-21T17:32:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:57 smithi157 ceph-mon[49460]: pgmap v218: 97 pgs: 5 active+undersized, 7 peering, 4 stale+active+clean, 10 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 611 op/s; 84/1566 objects degraded (5.364%) 2024-09-21T17:32:57.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:57 smithi157 podman[55153]: 2024-09-21 17:32:57.125490951 +0000 UTC m=+0.431995205 container init 8b9845976b0b628fd283ddd2f7269f2f9fd668d4eedaa60e95f18efd56b33f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T17:32:57.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:57 smithi157 podman[55153]: 2024-09-21 17:32:57.130390657 +0000 UTC m=+0.436894910 container start 8b9845976b0b628fd283ddd2f7269f2f9fd668d4eedaa60e95f18efd56b33f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:32:57.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:57 smithi157 podman[55153]: 2024-09-21 17:32:57.259250371 +0000 UTC m=+0.565754648 container attach 8b9845976b0b628fd283ddd2f7269f2f9fd668d4eedaa60e95f18efd56b33f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T17:32:57.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:57 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:32:57.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:57 smithi157 bash[55153]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:32:57.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:57 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:32:57.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:57 smithi157 bash[55153]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:32:57.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 218542 8.80 MB/sec execute 482 sec latency 69.536 ms 2024-09-21T17:32:58.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T17:32:58.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 bash[55153]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T17:32:58.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T17:32:58.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 bash[55153]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T17:32:58.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 218848 8.81 MB/sec execute 483 sec latency 51.486 ms 2024-09-21T17:32:59.028 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 bash[55153]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 bash[55153]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 bash[55153]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 bash[55153]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate[55168]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 bash[55153]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T17:32:59.029 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:58 smithi157 podman[55153]: 2024-09-21 17:32:58.702911609 +0000 UTC m=+2.009415854 container died 8b9845976b0b628fd283ddd2f7269f2f9fd668d4eedaa60e95f18efd56b33f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T17:32:59.255 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:32:59 smithi018 ceph-mon[63924]: pgmap v219: 97 pgs: 8 active+undersized, 7 peering, 16 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 535 op/s; 123/1533 objects degraded (8.023%) 2024-09-21T17:32:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:32:59 smithi157 ceph-mon[49460]: pgmap v219: 97 pgs: 8 active+undersized, 7 peering, 16 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 535 op/s; 123/1533 objects degraded (8.023%) 2024-09-21T17:32:59.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 219037 8.81 MB/sec execute 484 sec latency 76.406 ms 2024-09-21T17:33:00.100 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:32:59 smithi157 podman[55299]: 2024-09-21 17:32:59.830863213 +0000 UTC m=+1.119996297 container remove 8b9845976b0b628fd283ddd2f7269f2f9fd668d4eedaa60e95f18efd56b33f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:33:00.100 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:00 smithi157 podman[55358]: 2024-09-21 17:32:59.957595795 +0000 UTC m=+0.016875637 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:33:00.100 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:00 smithi157 podman[55358]: 2024-09-21 17:33:00.098427796 +0000 UTC m=+0.157707637 container create abad654e4600e3db95fc79720ae5049cfb7acc4e97867da362261962b108e478 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:33:00.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:00 smithi157 podman[55358]: 2024-09-21 17:33:00.384705988 +0000 UTC m=+0.443985833 container init abad654e4600e3db95fc79720ae5049cfb7acc4e97867da362261962b108e478 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:33:00.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:00 smithi157 podman[55358]: 2024-09-21 17:33:00.388324067 +0000 UTC m=+0.447603900 container start abad654e4600e3db95fc79720ae5049cfb7acc4e97867da362261962b108e478 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2) 2024-09-21T17:33:00.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:00 smithi157 ceph-osd[55378]: -- 172.21.15.157:0/1363245576 <== mon.1 v2:172.21.15.157:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56093ad101e0 con 0x560939eabc00 2024-09-21T17:33:00.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:00 smithi157 bash[55358]: abad654e4600e3db95fc79720ae5049cfb7acc4e97867da362261962b108e478 2024-09-21T17:33:00.778 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:00 smithi157 systemd[1]: Started Ceph osd.5 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:33:00.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 219239 8.81 MB/sec execute 485 sec latency 77.445 ms 2024-09-21T17:33:01.351 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:01 smithi018 ceph-mon[63924]: pgmap v220: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 2.3 MiB/s wr, 242 op/s; 174/1545 objects degraded (11.262%) 2024-09-21T17:33:01.351 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:01.352 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:01.352 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:33:01.352 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:01.352 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:33:01.458 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:01 smithi157 ceph-mon[49460]: pgmap v220: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 2.3 MiB/s wr, 242 op/s; 174/1545 objects degraded (11.262%) 2024-09-21T17:33:01.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:01.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:01.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:33:01.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:01.459 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:33:01.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 219516 8.80 MB/sec execute 486 sec latency 45.861 ms 2024-09-21T17:33:02.372 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:02 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 174/1545 objects degraded (11.262%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:02.428 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:02 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 174/1545 objects degraded (11.262%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:02.428 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:02 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:33:02.276+0000 7f463cfb7640 -1 Falling back to public interface 2024-09-21T17:33:02.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 219806 8.79 MB/sec execute 487 sec latency 97.825 ms 2024-09-21T17:33:03.387 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:03 smithi018 ceph-mon[63924]: pgmap v221: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.9 MiB/s wr, 378 op/s; 183/1614 objects degraded (11.338%) 2024-09-21T17:33:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:03 smithi157 ceph-mon[49460]: pgmap v221: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.9 MiB/s wr, 378 op/s; 183/1614 objects degraded (11.338%) 2024-09-21T17:33:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 220034 8.79 MB/sec execute 488 sec latency 43.289 ms 2024-09-21T17:33:04.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 220348 8.78 MB/sec execute 489 sec latency 87.892 ms 2024-09-21T17:33:05.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:05 smithi018 ceph-mon[63924]: pgmap v222: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.1 MiB/s wr, 410 op/s; 189/1680 objects degraded (11.250%) 2024-09-21T17:33:05.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:05.173 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:05 smithi157 ceph-mon[49460]: pgmap v222: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.1 MiB/s wr, 410 op/s; 189/1680 objects degraded (11.250%) 2024-09-21T17:33:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:05.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 220652 8.78 MB/sec execute 490 sec latency 63.872 ms 2024-09-21T17:33:06.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:06.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:06.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:06 smithi018 ceph-mon[63924]: pgmap v223: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.9 MiB/s wr, 370 op/s; 177/1635 objects degraded (10.826%) 2024-09-21T17:33:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:06.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:06 smithi157 ceph-mon[49460]: pgmap v223: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.9 MiB/s wr, 370 op/s; 177/1635 objects degraded (10.826%) 2024-09-21T17:33:06.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 221076 8.78 MB/sec execute 491 sec latency 56.231 ms 2024-09-21T17:33:07.496 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:07 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 177/1635 objects degraded (10.826%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:07 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 177/1635 objects degraded (10.826%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:07.779 INFO:teuthology.orchestra.run.smithi018.stdout: 5 221374 8.77 MB/sec execute 492 sec latency 46.892 ms 2024-09-21T17:33:08.509 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:08 smithi018 ceph-mon[63924]: pgmap v224: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.7 MiB/s wr, 355 op/s; 174/1584 objects degraded (10.985%) 2024-09-21T17:33:08.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:08 smithi157 ceph-mon[49460]: pgmap v224: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.7 MiB/s wr, 355 op/s; 174/1584 objects degraded (10.985%) 2024-09-21T17:33:08.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 221847 8.79 MB/sec execute 493 sec latency 50.181 ms 2024-09-21T17:33:09.028 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:08 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:33:08.545+0000 7f463cfb7640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-21T17:33:09.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 222255 8.79 MB/sec execute 494 sec latency 43.897 ms 2024-09-21T17:33:10.278 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:09 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:33:09.816+0000 7f463cfb7640 -1 osd.5 124 log_to_monitors true 2024-09-21T17:33:10.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 222661 8.81 MB/sec execute 495 sec latency 42.645 ms 2024-09-21T17:33:10.794 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:10.794 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='osd.5 [v2:172.21.15.157:6808/2519477248,v1:172.21.15.157:6809/2519477248]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: pgmap v225: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 370 op/s; 165/1524 objects degraded (10.827%) 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T17:33:10.795 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:10 smithi018 ceph-mon[63924]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T17:33:11.028 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:33:10.639+0000 7f4634d61640 -1 osd.5 124 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:33:11.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:11.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='osd.5 [v2:172.21.15.157:6808/2519477248,v1:172.21.15.157:6809/2519477248]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: pgmap v225: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 370 op/s; 165/1524 objects degraded (10.827%) 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T17:33:11.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:10 smithi157 ceph-mon[49460]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T17:33:11.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 222981 8.81 MB/sec execute 496 sec latency 99.442 ms 2024-09-21T17:33:11.815 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:11 smithi018 ceph-mon[63924]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T17:33:11.815 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:11 smithi018 ceph-mon[63924]: osdmap e127: 8 total, 7 up, 8 in 2024-09-21T17:33:11.815 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:11 smithi018 ceph-mon[63924]: from='osd.5 [v2:172.21.15.157:6808/2519477248,v1:172.21.15.157:6809/2519477248]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:33:11.815 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:11 smithi018 ceph-mon[63924]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:33:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:11 smithi157 ceph-mon[49460]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T17:33:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:11 smithi157 ceph-mon[49460]: osdmap e127: 8 total, 7 up, 8 in 2024-09-21T17:33:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:11 smithi157 ceph-mon[49460]: from='osd.5 [v2:172.21.15.157:6808/2519477248,v1:172.21.15.157:6809/2519477248]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:33:12.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:11 smithi157 ceph-mon[49460]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:33:12.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 223186 8.80 MB/sec execute 497 sec latency 1099.472 ms 2024-09-21T17:33:12.826 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:12 smithi018 ceph-mon[63924]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:33:12.826 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:12 smithi018 ceph-mon[63924]: osd.5 [v2:172.21.15.157:6808/2519477248,v1:172.21.15.157:6809/2519477248] boot 2024-09-21T17:33:12.827 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:12 smithi018 ceph-mon[63924]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T17:33:12.827 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:12 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T17:33:12.827 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:12 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 165/1524 objects degraded (10.827%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:12.827 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:12 smithi018 ceph-mon[63924]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T17:33:12.827 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:12 smithi018 ceph-mon[63924]: pgmap v229: 97 pgs: 1 remapped+peering, 7 active+undersized, 22 active+undersized+degraded, 67 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.0 MiB/s wr, 640 op/s; 172/1656 objects degraded (10.386%) 2024-09-21T17:33:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:12 smithi157 ceph-mon[49460]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:33:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:12 smithi157 ceph-mon[49460]: osd.5 [v2:172.21.15.157:6808/2519477248,v1:172.21.15.157:6809/2519477248] boot 2024-09-21T17:33:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:12 smithi157 ceph-mon[49460]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T17:33:13.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:12 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T17:33:13.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:12 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 165/1524 objects degraded (10.827%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:13.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:12 smithi157 ceph-mon[49460]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T17:33:13.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:12 smithi157 ceph-mon[49460]: pgmap v229: 97 pgs: 1 remapped+peering, 7 active+undersized, 22 active+undersized+degraded, 67 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.0 MiB/s wr, 640 op/s; 172/1656 objects degraded (10.386%) 2024-09-21T17:33:13.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 223414 8.79 MB/sec execute 498 sec latency 2012.700 ms 2024-09-21T17:33:13.840 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:13 smithi018 ceph-mon[63924]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T17:33:14.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:13 smithi157 ceph-mon[49460]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T17:33:14.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 223748 8.80 MB/sec execute 499 sec latency 833.651 ms 2024-09-21T17:33:14.886 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:14 smithi018 ceph-mon[63924]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T17:33:14.886 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:14 smithi018 ceph-mon[63924]: pgmap v232: 97 pgs: 2 remapped+peering, 7 active+undersized, 19 active+undersized+degraded, 69 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.5 MiB/s wr, 695 op/s; 156/1683 objects degraded (9.269%) 2024-09-21T17:33:14.886 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:14 smithi018 ceph-mon[63924]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T17:33:15.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:14 smithi157 ceph-mon[49460]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T17:33:15.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:14 smithi157 ceph-mon[49460]: pgmap v232: 97 pgs: 2 remapped+peering, 7 active+undersized, 19 active+undersized+degraded, 69 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.5 MiB/s wr, 695 op/s; 156/1683 objects degraded (9.269%) 2024-09-21T17:33:15.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:14 smithi157 ceph-mon[49460]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T17:33:15.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 224056 8.79 MB/sec execute 500 sec latency 828.927 ms 2024-09-21T17:33:16.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:16.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:33:16.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:16 smithi018 ceph-mon[63924]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T17:33:16.649 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:16 smithi018 ceph-mon[63924]: pgmap v235: 97 pgs: 1 remapped+peering, 2 active+undersized, 2 active+undersized+degraded, 92 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 366 op/s; 12/1692 objects degraded (0.709%); 1.3 MiB/s, 43 objects/s recovering 2024-09-21T17:33:16.780 INFO:teuthology.orchestra.run.smithi018.stdout: 5 224435 8.78 MB/sec execute 501 sec latency 30.675 ms 2024-09-21T17:33:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:33:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:16 smithi157 ceph-mon[49460]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T17:33:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:16 smithi157 ceph-mon[49460]: pgmap v235: 97 pgs: 1 remapped+peering, 2 active+undersized, 2 active+undersized+degraded, 92 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 366 op/s; 12/1692 objects degraded (0.709%); 1.3 MiB/s, 43 objects/s recovering 2024-09-21T17:33:17.685 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:17 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 156/1683 objects degraded (9.269%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:17.786 INFO:teuthology.orchestra.run.smithi018.stdout: 5 224727 8.78 MB/sec execute 502 sec latency 59.775 ms 2024-09-21T17:33:18.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:17 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 156/1683 objects degraded (9.269%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:18.734 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:18 smithi018 ceph-mon[63924]: pgmap v236: 97 pgs: 1 remapped+peering, 96 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 450 op/s; 1.1 MiB/s, 34 objects/s recovering 2024-09-21T17:33:18.786 INFO:teuthology.orchestra.run.smithi018.stdout: 5 225025 8.78 MB/sec execute 503 sec latency 50.741 ms 2024-09-21T17:33:19.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:18 smithi157 ceph-mon[49460]: pgmap v236: 97 pgs: 1 remapped+peering, 96 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 450 op/s; 1.1 MiB/s, 34 objects/s recovering 2024-09-21T17:33:19.756 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:19 smithi018 ceph-mon[63924]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/1692 objects degraded (0.709%), 2 pgs degraded) 2024-09-21T17:33:19.757 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:19 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:33:19.786 INFO:teuthology.orchestra.run.smithi018.stdout: 5 225394 8.79 MB/sec execute 504 sec latency 36.597 ms 2024-09-21T17:33:20.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:19 smithi157 ceph-mon[49460]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/1692 objects degraded (0.709%), 2 pgs degraded) 2024-09-21T17:33:20.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:19 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:33:20.773 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:20 smithi018 ceph-mon[63924]: pgmap v237: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.1 MiB/s wr, 374 op/s; 1.1 MiB/s, 32 objects/s recovering 2024-09-21T17:33:20.786 INFO:teuthology.orchestra.run.smithi018.stdout: 5 225696 8.80 MB/sec execute 505 sec latency 44.962 ms 2024-09-21T17:33:21.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:20 smithi157 ceph-mon[49460]: pgmap v237: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.1 MiB/s wr, 374 op/s; 1.1 MiB/s, 32 objects/s recovering 2024-09-21T17:33:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: 5 225957 8.79 MB/sec execute 506 sec latency 64.373 ms 2024-09-21T17:33:22.787 INFO:teuthology.orchestra.run.smithi018.stdout: 5 226249 8.79 MB/sec execute 507 sec latency 46.765 ms 2024-09-21T17:33:23.244 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:23 smithi018 ceph-mon[63924]: pgmap v238: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.1 MiB/s wr, 534 op/s; 883 KiB/s, 25 objects/s recovering 2024-09-21T17:33:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:23 smithi157 ceph-mon[49460]: pgmap v238: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.1 MiB/s wr, 534 op/s; 883 KiB/s, 25 objects/s recovering 2024-09-21T17:33:23.787 INFO:teuthology.orchestra.run.smithi018.stdout: 5 226471 8.78 MB/sec execute 508 sec latency 45.054 ms 2024-09-21T17:33:24.637 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:33:24.787 INFO:teuthology.orchestra.run.smithi018.stdout: 5 226766 8.78 MB/sec execute 509 sec latency 64.832 ms 2024-09-21T17:33:25.200 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:25 smithi018 ceph-mon[63924]: pgmap v239: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 750 KiB/s, 22 objects/s recovering 2024-09-21T17:33:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:25 smithi157 ceph-mon[49460]: pgmap v239: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 750 KiB/s, 22 objects/s recovering 2024-09-21T17:33:25.654 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (8m) 100s ago 17m 19.7M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (4m) 100s ago 17m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (4m) 21s ago 16m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (7m) 100s ago 17m 47.5M - dad864ee21e9 48d78bd31599 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (11m) 100s ago 11m 38.4M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (11m) 21s ago 11m 16.6M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (7m) 100s ago 18m 520M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (6m) 21s ago 16m 458M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (6m) 100s ago 18m 51.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (5m) 21s ago 16m 43.9M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:33:25.655 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (10m) 100s ago 10m 276M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (10m) 21s ago 10m 87.4M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (7m) 100s ago 16m 10.1M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (7m) 21s ago 16m 12.1M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (4m) 100s ago 15m 337M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (3m) 100s ago 15m 272M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (2m) 100s ago 14m 218M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (108s) 100s ago 14m 13.3M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (63s) 21s ago 13m 239M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 2552dce6837e 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (25s) 21s ago 13m 11.4M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d abad654e4600 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (13m) 21s ago 13m 438M 2704M 17.2.0 e1d6a67b021e dbd6d8c6c19a 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (12m) 21s ago 12m 674M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:33:25.656 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (6m) 100s ago 16m 53.1M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:33:25.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 226975 8.77 MB/sec execute 510 sec latency 85.134 ms 2024-09-21T17:33:26.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:26 smithi018 ceph-mon[63924]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:26.210 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:26 smithi018 ceph-mon[63924]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:26.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:26 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T17:33:26.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:26 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T17:33:26.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:26 smithi018 ceph-mon[63924]: Upgrade: osd.6 is safe to restart 2024-09-21T17:33:26.211 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:26 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:26.369 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:33:26.369 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:33:26.369 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:33:26.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:33:26.369 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:33:26.369 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:33:26.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:33:26.369 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 6 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 10 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:33:26.370 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:33:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:26 smithi157 ceph-mon[49460]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:26 smithi157 ceph-mon[49460]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:26 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T17:33:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:26 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T17:33:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:26 smithi157 ceph-mon[49460]: Upgrade: osd.6 is safe to restart 2024-09-21T17:33:26.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:26 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:26.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 227198 8.77 MB/sec execute 511 sec latency 69.063 ms 2024-09-21T17:33:26.970 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:33:26.970 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "12/23 daemons upgraded", 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:33:26.971 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:33:27.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:27 smithi018 ceph-mon[63924]: from='client.44263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:27.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:27 smithi018 ceph-mon[63924]: pgmap v240: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 514 op/s; 235 KiB/s, 11 objects/s recovering 2024-09-21T17:33:27.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:27 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1968684992' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:27.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:27.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T17:33:27.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:33:27.298 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:27 smithi157 ceph-mon[49460]: from='client.44263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:27.298 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:27 smithi157 ceph-mon[49460]: pgmap v240: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 514 op/s; 235 KiB/s, 11 objects/s recovering 2024-09-21T17:33:27.298 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:27 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1968684992' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:27.298 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:27.298 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T17:33:27.298 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:33:27.689 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:33:27.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 227528 8.77 MB/sec execute 512 sec latency 50.311 ms 2024-09-21T17:33:28.247 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:28 smithi018 ceph-mon[63924]: Upgrade: Updating osd.6 2024-09-21T17:33:28.247 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:28 smithi018 ceph-mon[63924]: Deploying daemon osd.6 on smithi157 2024-09-21T17:33:28.247 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:28 smithi018 ceph-mon[63924]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:28.248 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:28 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3834009647' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:33:28.317 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:28 smithi157 ceph-mon[49460]: Upgrade: Updating osd.6 2024-09-21T17:33:28.317 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:28 smithi157 ceph-mon[49460]: Deploying daemon osd.6 on smithi157 2024-09-21T17:33:28.317 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:28 smithi157 ceph-mon[49460]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:28.318 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:28 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3834009647' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:33:28.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 227829 8.76 MB/sec execute 513 sec latency 55.191 ms 2024-09-21T17:33:29.269 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:29 smithi018 ceph-mon[63924]: pgmap v241: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 362 op/s; 136 KiB/s, 2 objects/s recovering 2024-09-21T17:33:29.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:29 smithi157 ceph-mon[49460]: pgmap v241: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 362 op/s; 136 KiB/s, 2 objects/s recovering 2024-09-21T17:33:29.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 228095 8.75 MB/sec execute 514 sec latency 51.915 ms 2024-09-21T17:33:30.778 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:30 smithi157 systemd[1]: Stopping Ceph osd.6 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:33:30.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 228362 8.75 MB/sec execute 515 sec latency 47.348 ms 2024-09-21T17:33:31.158 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[35974]: 2024-09-21T17:33:30.785+0000 7f9ef7145700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:33:31.158 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[35974]: 2024-09-21T17:33:30.785+0000 7f9ef7145700 -1 osd.6 133 *** Got signal Terminated *** 2024-09-21T17:33:31.158 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[35974]: 2024-09-21T17:33:30.785+0000 7f9ef7145700 -1 osd.6 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:33:31.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:31 smithi018 ceph-mon[63924]: pgmap v242: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 83 KiB/s, 1 objects/s recovering 2024-09-21T17:33:31.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:31.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:33:31.359 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:31 smithi018 ceph-mon[63924]: osd.6 marked itself down and dead 2024-09-21T17:33:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:31 smithi157 ceph-mon[49460]: pgmap v242: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 83 KiB/s, 1 objects/s recovering 2024-09-21T17:33:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:33:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:31 smithi157 ceph-mon[49460]: osd.6 marked itself down and dead 2024-09-21T17:33:31.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 228656 8.75 MB/sec execute 516 sec latency 216.697 ms 2024-09-21T17:33:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:32 smithi157 ceph-mon[49460]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:33:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:32 smithi157 ceph-mon[49460]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T17:33:32.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:32 smithi157 podman[56930]: 2024-09-21 17:33:32.211355072 +0000 UTC m=+1.559582989 container died dbd6d8c6c19a7a34976c5a0342e3267c200f9a04990b27a7ec38a908ff4f5225 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, architecture=x86_64, vendor=Red Hat, Inc., ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, version=8) 2024-09-21T17:33:32.616 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:32 smithi018 ceph-mon[63924]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:33:32.616 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:32 smithi018 ceph-mon[63924]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T17:33:32.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 228732 8.74 MB/sec execute 517 sec latency 1043.085 ms 2024-09-21T17:33:32.979 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:32 smithi157 podman[56930]: 2024-09-21 17:33:32.742613687 +0000 UTC m=+2.090841591 container cleanup dbd6d8c6c19a7a34976c5a0342e3267c200f9a04990b27a7ec38a908ff4f5225 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, ceph=True, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.expose-services=, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T17:33:32.980 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:32 smithi157 bash[56930]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6 2024-09-21T17:33:33.278 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:32 smithi157 podman[56946]: 2024-09-21 17:33:32.977686587 +0000 UTC m=+0.767007876 container remove dbd6d8c6c19a7a34976c5a0342e3267c200f9a04990b27a7ec38a908ff4f5225 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, ceph=True, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, distribution-scope=public, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , version=8, CEPH_POINT_RELEASE=-17.2.0) 2024-09-21T17:33:33.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:33 smithi157 ceph-mon[49460]: pgmap v244: 97 pgs: 13 peering, 6 stale+active+clean, 78 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-09-21T17:33:33.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:33 smithi157 ceph-mon[49460]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T17:33:33.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:33 smithi157 ceph-mon[49460]: osdmap e135: 8 total, 7 up, 8 in 2024-09-21T17:33:33.381 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:33 smithi018 ceph-mon[63924]: pgmap v244: 97 pgs: 13 peering, 6 stale+active+clean, 78 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-09-21T17:33:33.381 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:33 smithi018 ceph-mon[63924]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T17:33:33.381 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:33 smithi018 ceph-mon[63924]: osdmap e135: 8 total, 7 up, 8 in 2024-09-21T17:33:33.778 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:33 smithi157 podman[57031]: 2024-09-21 17:33:33.348714743 +0000 UTC m=+0.017745790 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:33:33.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 229154 8.75 MB/sec execute 518 sec latency 45.992 ms 2024-09-21T17:33:34.206 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:33 smithi157 podman[57031]: 2024-09-21 17:33:33.954276331 +0000 UTC m=+0.623307352 container create 965a16e83424e8dd0aa3e931b513e0358afd537a55e8da399c4630afd38f38f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T17:33:34.207 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:34 smithi157 podman[57031]: 2024-09-21 17:33:34.201280913 +0000 UTC m=+0.870311942 container init 965a16e83424e8dd0aa3e931b513e0358afd537a55e8da399c4630afd38f38f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:33:34.396 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:34 smithi018 ceph-mon[63924]: Health check failed: Degraded data redundancy: 35/1536 objects degraded (2.279%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:34.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:34 smithi157 ceph-mon[49460]: Health check failed: Degraded data redundancy: 35/1536 objects degraded (2.279%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:34.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:34 smithi157 podman[57031]: 2024-09-21 17:33:34.205072613 +0000 UTC m=+0.874103639 container start 965a16e83424e8dd0aa3e931b513e0358afd537a55e8da399c4630afd38f38f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:33:34.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:34 smithi157 podman[57031]: 2024-09-21 17:33:34.31941276 +0000 UTC m=+0.988443788 container attach 965a16e83424e8dd0aa3e931b513e0358afd537a55e8da399c4630afd38f38f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T17:33:34.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:34 smithi157 podman[57031]: 2024-09-21 17:33:34.4205488 +0000 UTC m=+1.089579835 container died 965a16e83424e8dd0aa3e931b513e0358afd537a55e8da399c4630afd38f38f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:33:34.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 229465 8.75 MB/sec execute 519 sec latency 45.709 ms 2024-09-21T17:33:35.409 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:35 smithi018 ceph-mon[63924]: pgmap v246: 97 pgs: 2 active+undersized, 13 peering, 4 stale+active+clean, 6 active+undersized+degraded, 72 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.6 MiB/s wr, 408 op/s; 35/1536 objects degraded (2.279%) 2024-09-21T17:33:35.503 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:35 smithi157 ceph-mon[49460]: pgmap v246: 97 pgs: 2 active+undersized, 13 peering, 4 stale+active+clean, 6 active+undersized+degraded, 72 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.6 MiB/s wr, 408 op/s; 35/1536 objects degraded (2.279%) 2024-09-21T17:33:35.504 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:35 smithi157 podman[57055]: 2024-09-21 17:33:35.243659848 +0000 UTC m=+0.817467139 container remove 965a16e83424e8dd0aa3e931b513e0358afd537a55e8da399c4630afd38f38f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T17:33:35.504 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:35 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-21T17:33:35.504 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:35 smithi157 systemd[1]: Stopped Ceph osd.6 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:33:35.504 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:35 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.6.service: Consumed 3min 47.244s CPU time. 2024-09-21T17:33:35.774 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:35 smithi157 systemd[1]: Starting Ceph osd.6 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:33:35.774 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:35 smithi157 podman[57152]: 2024-09-21 17:33:35.625306739 +0000 UTC m=+0.016667269 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:33:35.788 INFO:teuthology.orchestra.run.smithi018.stdout: 5 229755 8.75 MB/sec execute 520 sec latency 51.887 ms 2024-09-21T17:33:36.028 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:35 smithi157 podman[57152]: 2024-09-21 17:33:35.772600086 +0000 UTC m=+0.163960612 container create 47985185c4a03a48a62203adff0167c1ae5c02837e38f7d1c86fd9344cd9d247 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:33:36.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 podman[57152]: 2024-09-21 17:33:36.034437499 +0000 UTC m=+0.425798036 container init 47985185c4a03a48a62203adff0167c1ae5c02837e38f7d1c86fd9344cd9d247 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T17:33:36.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 podman[57152]: 2024-09-21 17:33:36.037849305 +0000 UTC m=+0.429209840 container start 47985185c4a03a48a62203adff0167c1ae5c02837e38f7d1c86fd9344cd9d247 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-21T17:33:36.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 podman[57152]: 2024-09-21 17:33:36.134649514 +0000 UTC m=+0.526010084 container attach 47985185c4a03a48a62203adff0167c1ae5c02837e38f7d1c86fd9344cd9d247 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:33:36.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:33:36.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 bash[57152]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:33:36.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:33:36.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 bash[57152]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:33:36.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 229996 8.74 MB/sec execute 521 sec latency 78.391 ms 2024-09-21T17:33:37.271 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:37 smithi157 ceph-mon[49460]: pgmap v247: 97 pgs: 5 active+undersized, 13 peering, 17 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 435 op/s; 132/1605 objects degraded (8.224%) 2024-09-21T17:33:37.271 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T17:33:37.271 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 bash[57152]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T17:33:37.271 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T17:33:37.271 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:36 smithi157 bash[57152]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T17:33:37.430 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:37 smithi018 ceph-mon[63924]: pgmap v247: 97 pgs: 5 active+undersized, 13 peering, 17 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 435 op/s; 132/1605 objects degraded (8.224%) 2024-09-21T17:33:37.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 bash[57152]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 bash[57152]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 bash[57152]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 bash[57152]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate[57167]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 bash[57152]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-21T17:33:37.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:37 smithi157 podman[57152]: 2024-09-21 17:33:37.316292156 +0000 UTC m=+1.707652683 container died 47985185c4a03a48a62203adff0167c1ae5c02837e38f7d1c86fd9344cd9d247 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T17:33:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 230317 8.75 MB/sec execute 522 sec latency 38.241 ms 2024-09-21T17:33:38.269 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:38 smithi157 podman[57308]: 2024-09-21 17:33:38.016185724 +0000 UTC m=+0.692450301 container remove 47985185c4a03a48a62203adff0167c1ae5c02837e38f7d1c86fd9344cd9d247 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T17:33:38.269 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:38 smithi157 podman[57364]: 2024-09-21 17:33:38.137200542 +0000 UTC m=+0.017095660 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:33:38.446 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:38 smithi018 ceph-mon[63924]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T17:33:38.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:38 smithi157 ceph-mon[49460]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T17:33:38.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:38 smithi157 podman[57364]: 2024-09-21 17:33:38.267379114 +0000 UTC m=+0.147274231 container create fd2e6eb21352c5bc7b7470d37bb5d51eb3372ad690b728112c9ca5cc2e6ac98f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T17:33:38.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 230540 8.74 MB/sec execute 523 sec latency 61.887 ms 2024-09-21T17:33:39.028 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:38 smithi157 podman[57364]: 2024-09-21 17:33:38.553828542 +0000 UTC m=+0.433723675 container init fd2e6eb21352c5bc7b7470d37bb5d51eb3372ad690b728112c9ca5cc2e6ac98f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T17:33:39.028 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:38 smithi157 podman[57364]: 2024-09-21 17:33:38.558265098 +0000 UTC m=+0.438160210 container start fd2e6eb21352c5bc7b7470d37bb5d51eb3372ad690b728112c9ca5cc2e6ac98f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T17:33:39.029 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:38 smithi157 ceph-osd[57383]: -- 172.21.15.157:0/3789400194 <== mon.1 v2:172.21.15.157:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d40bcaa1e0 con 0x55d40bc9c000 2024-09-21T17:33:39.029 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:38 smithi157 bash[57364]: fd2e6eb21352c5bc7b7470d37bb5d51eb3372ad690b728112c9ca5cc2e6ac98f 2024-09-21T17:33:39.029 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:38 smithi157 systemd[1]: Started Ceph osd.6 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:33:39.479 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:39 smithi018 ceph-mon[63924]: pgmap v248: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.4 MiB/s wr, 430 op/s; 195/1617 objects degraded (12.059%); 257 B/s, 0 objects/s recovering 2024-09-21T17:33:39.479 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:39.479 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:39.479 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:33:39.609 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:39 smithi157 ceph-mon[49460]: pgmap v248: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.4 MiB/s wr, 430 op/s; 195/1617 objects degraded (12.059%); 257 B/s, 0 objects/s recovering 2024-09-21T17:33:39.609 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:39.609 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:39.610 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:33:39.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 230800 8.74 MB/sec execute 524 sec latency 57.549 ms 2024-09-21T17:33:40.516 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:40 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 200/1653 objects degraded (12.099%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:40.517 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:40 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:33:40.433+0000 7f60021e4640 -1 Falling back to public interface 2024-09-21T17:33:40.517 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:40 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 200/1653 objects degraded (12.099%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:40.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 231111 8.74 MB/sec execute 525 sec latency 50.008 ms 2024-09-21T17:33:41.535 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:41 smithi018 ceph-mon[63924]: pgmap v249: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.8 MiB/s wr, 374 op/s; 200/1653 objects degraded (12.099%); 242 B/s, 0 objects/s recovering 2024-09-21T17:33:41.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:41 smithi157 ceph-mon[49460]: pgmap v249: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.8 MiB/s wr, 374 op/s; 200/1653 objects degraded (12.099%); 242 B/s, 0 objects/s recovering 2024-09-21T17:33:41.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 231410 8.72 MB/sec execute 526 sec latency 53.402 ms 2024-09-21T17:33:42.561 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:42 smithi018 ceph-mon[63924]: pgmap v250: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 400 op/s; 191/1626 objects degraded (11.747%); 206 B/s, 0 objects/s recovering 2024-09-21T17:33:42.659 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:42 smithi157 ceph-mon[49460]: pgmap v250: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 400 op/s; 191/1626 objects degraded (11.747%); 206 B/s, 0 objects/s recovering 2024-09-21T17:33:42.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 231727 8.72 MB/sec execute 527 sec latency 59.641 ms 2024-09-21T17:33:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 231949 8.72 MB/sec execute 528 sec latency 52.828 ms 2024-09-21T17:33:44.355 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:44.355 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:44.355 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:44.356 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:44 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:44.439 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:44 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:44.439 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:44 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:44.440 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:44 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:44.440 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:44 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:44.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 232322 8.73 MB/sec execute 529 sec latency 60.631 ms 2024-09-21T17:33:45.389 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:45 smithi018 ceph-mon[63924]: pgmap v251: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.0 MiB/s wr, 404 op/s; 193/1620 objects degraded (11.914%); 179 B/s, 0 objects/s recovering 2024-09-21T17:33:45.490 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:45 smithi157 ceph-mon[49460]: pgmap v251: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.0 MiB/s wr, 404 op/s; 193/1620 objects degraded (11.914%); 179 B/s, 0 objects/s recovering 2024-09-21T17:33:45.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 232683 8.73 MB/sec execute 530 sec latency 54.234 ms 2024-09-21T17:33:46.681 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:46.681 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:33:46.681 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:46 smithi018 ceph-mon[63924]: pgmap v252: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 421 op/s; 196/1596 objects degraded (12.281%); 171 B/s, 0 objects/s recovering 2024-09-21T17:33:46.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:46.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:33:46.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:46 smithi157 ceph-mon[49460]: pgmap v252: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 421 op/s; 196/1596 objects degraded (12.281%); 171 B/s, 0 objects/s recovering 2024-09-21T17:33:46.779 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:33:46.416+0000 7f60021e4640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-21T17:33:46.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 232927 8.73 MB/sec execute 531 sec latency 61.980 ms 2024-09-21T17:33:47.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:47 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 196/1596 objects degraded (12.281%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:47.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:47 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:47.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:47 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:47.696 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:47 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:33:47.697 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:47 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:33:47.697 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:47 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:47.697 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:47 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:47.778 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:47 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:33:47.649+0000 7f60021e4640 -1 osd.6 133 log_to_monitors true 2024-09-21T17:33:47.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:47 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 196/1596 objects degraded (12.281%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:47.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:47 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:47.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:47 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:47.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:47 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:33:47.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:47 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:33:47.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:47 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:47.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:47 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:33:47.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 233230 8.73 MB/sec execute 532 sec latency 61.469 ms 2024-09-21T17:33:48.717 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: from='osd.6 [v2:172.21.15.157:6816/3057171179,v1:172.21.15.157:6817/3057171179]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: pgmap v253: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 408 op/s; 196/1593 objects degraded (12.304%); 171 B/s, 0 objects/s recovering 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T17:33:48.718 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:48 smithi018 ceph-mon[63924]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T17:33:48.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 233445 8.72 MB/sec execute 533 sec latency 79.184 ms 2024-09-21T17:33:49.028 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:33:48.641+0000 7f5ff9f8e640 -1 osd.6 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:33:49.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: from='osd.6 [v2:172.21.15.157:6816/3057171179,v1:172.21.15.157:6817/3057171179]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: pgmap v253: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 408 op/s; 196/1593 objects degraded (12.304%); 171 B/s, 0 objects/s recovering 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T17:33:49.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:48 smithi157 ceph-mon[49460]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T17:33:49.752 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:49 smithi018 ceph-mon[63924]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T17:33:49.753 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:49 smithi018 ceph-mon[63924]: osdmap e136: 8 total, 7 up, 8 in 2024-09-21T17:33:49.753 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:49 smithi018 ceph-mon[63924]: from='osd.6 [v2:172.21.15.157:6816/3057171179,v1:172.21.15.157:6817/3057171179]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:33:49.753 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:49 smithi018 ceph-mon[63924]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:33:49.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 233764 8.72 MB/sec execute 534 sec latency 173.953 ms 2024-09-21T17:33:50.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:49 smithi157 ceph-mon[49460]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T17:33:50.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:49 smithi157 ceph-mon[49460]: osdmap e136: 8 total, 7 up, 8 in 2024-09-21T17:33:50.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:49 smithi157 ceph-mon[49460]: from='osd.6 [v2:172.21.15.157:6816/3057171179,v1:172.21.15.157:6817/3057171179]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:33:50.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:49 smithi157 ceph-mon[49460]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:33:50.775 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:50 smithi018 ceph-mon[63924]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:33:50.775 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:50 smithi018 ceph-mon[63924]: osd.6 [v2:172.21.15.157:6816/3057171179,v1:172.21.15.157:6817/3057171179] boot 2024-09-21T17:33:50.775 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:50 smithi018 ceph-mon[63924]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T17:33:50.776 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:50 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T17:33:50.776 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:50 smithi018 ceph-mon[63924]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.5 MiB/s wr, 319 op/s; 197/1623 objects degraded (12.138%) 2024-09-21T17:33:50.776 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:50 smithi018 ceph-mon[63924]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T17:33:50.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 233958 8.71 MB/sec execute 535 sec latency 1087.444 ms 2024-09-21T17:33:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:50 smithi157 ceph-mon[49460]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:33:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:50 smithi157 ceph-mon[49460]: osd.6 [v2:172.21.15.157:6816/3057171179,v1:172.21.15.157:6817/3057171179] boot 2024-09-21T17:33:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:50 smithi157 ceph-mon[49460]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T17:33:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:50 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T17:33:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:50 smithi157 ceph-mon[49460]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.5 MiB/s wr, 319 op/s; 197/1623 objects degraded (12.138%) 2024-09-21T17:33:51.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:50 smithi157 ceph-mon[49460]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T17:33:51.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 234058 8.70 MB/sec execute 536 sec latency 1811.451 ms 2024-09-21T17:33:52.698 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:52 smithi018 ceph-mon[63924]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T17:33:52.698 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:52 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 197/1623 objects degraded (12.138%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:52.699 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:52 smithi018 ceph-mon[63924]: pgmap v259: 97 pgs: 4 activating, 3 remapped+peering, 1 activating+degraded, 3 active+undersized, 13 active+undersized+degraded, 73 active+clean; 344 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 124/1614 objects degraded (7.683%); 76 KiB/s, 14 objects/s recovering 2024-09-21T17:33:52.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 234493 8.71 MB/sec execute 537 sec latency 32.787 ms 2024-09-21T17:33:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:52 smithi157 ceph-mon[49460]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T17:33:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:52 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 197/1623 objects degraded (12.138%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T17:33:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:52 smithi157 ceph-mon[49460]: pgmap v259: 97 pgs: 4 activating, 3 remapped+peering, 1 activating+degraded, 3 active+undersized, 13 active+undersized+degraded, 73 active+clean; 344 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 124/1614 objects degraded (7.683%); 76 KiB/s, 14 objects/s recovering 2024-09-21T17:33:53.703 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:53 smithi018 ceph-mon[63924]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T17:33:53.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 234874 8.71 MB/sec execute 538 sec latency 410.129 ms 2024-09-21T17:33:54.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:53 smithi157 ceph-mon[49460]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T17:33:54.733 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:54 smithi018 ceph-mon[63924]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T17:33:54.733 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:54 smithi018 ceph-mon[63924]: pgmap v262: 97 pgs: 1 peering, 4 activating, 3 remapped+peering, 1 activating+degraded, 88 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.4 MiB/s wr, 632 op/s; 42/1599 objects degraded (2.627%); 727 KiB/s, 33 objects/s recovering 2024-09-21T17:33:54.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 235136 8.70 MB/sec execute 539 sec latency 67.066 ms 2024-09-21T17:33:55.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:54 smithi157 ceph-mon[49460]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T17:33:55.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:54 smithi157 ceph-mon[49460]: pgmap v262: 97 pgs: 1 peering, 4 activating, 3 remapped+peering, 1 activating+degraded, 88 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.4 MiB/s wr, 632 op/s; 42/1599 objects degraded (2.627%); 727 KiB/s, 33 objects/s recovering 2024-09-21T17:33:55.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 235455 8.70 MB/sec execute 540 sec latency 59.329 ms 2024-09-21T17:33:56.362 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:56 smithi018 ceph-mon[63924]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1599 objects degraded (2.627%), 1 pg degraded) 2024-09-21T17:33:56.362 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:56 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:33:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:56 smithi157 ceph-mon[49460]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1599 objects degraded (2.627%), 1 pg degraded) 2024-09-21T17:33:56.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:56 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:33:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 235845 8.71 MB/sec execute 541 sec latency 52.886 ms 2024-09-21T17:33:57.380 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:57 smithi018 ceph-mon[63924]: pgmap v263: 97 pgs: 1 peering, 2 remapped+peering, 94 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 536 op/s; 1.4 MiB/s, 32 objects/s recovering 2024-09-21T17:33:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:57 smithi157 ceph-mon[49460]: pgmap v263: 97 pgs: 1 peering, 2 remapped+peering, 94 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 536 op/s; 1.4 MiB/s, 32 objects/s recovering 2024-09-21T17:33:57.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 236207 8.71 MB/sec execute 542 sec latency 56.563 ms 2024-09-21T17:33:58.252 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:33:58.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 236480 8.72 MB/sec execute 543 sec latency 54.918 ms 2024-09-21T17:33:59.241 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:33:59.241 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (8m) 2m ago 17m 19.7M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:33:59.241 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (5m) 2m ago 17m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:33:59.241 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (5m) 16s ago 17m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:33:59.241 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (8m) 2m ago 17m 47.5M - dad864ee21e9 48d78bd31599 2024-09-21T17:33:59.241 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (11m) 2m ago 11m 38.4M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:33:59.241 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (11m) 16s ago 11m 16.7M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:33:59.241 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (7m) 2m ago 19m 520M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (6m) 16s ago 17m 459M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (6m) 2m ago 19m 51.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (6m) 16s ago 17m 45.2M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (11m) 2m ago 11m 276M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (11m) 16s ago 11m 87.4M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (8m) 2m ago 17m 10.1M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (7m) 16s ago 16m 12.1M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (4m) 2m ago 16m 337M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (3m) 2m ago 15m 272M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (3m) 2m ago 15m 218M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (2m) 2m ago 14m 13.3M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (97s) 16s ago 14m 296M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 2552dce6837e 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (59s) 16s ago 14m 234M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d abad654e4600 2024-09-21T17:33:59.242 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (21s) 16s ago 13m 11.4M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d fd2e6eb21352 2024-09-21T17:33:59.243 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (13m) 16s ago 13m 714M 2704M 17.2.0 e1d6a67b021e b08f1b06fb67 2024-09-21T17:33:59.243 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (7m) 2m ago 16m 53.1M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:33:59.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:59 smithi018 ceph-mon[63924]: pgmap v264: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 457 op/s; 1.5 MiB/s, 33 keys/s, 27 objects/s recovering 2024-09-21T17:33:59.424 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:33:59 smithi018 ceph-mon[63924]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:59 smithi157 ceph-mon[49460]: pgmap v264: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 457 op/s; 1.5 MiB/s, 33 keys/s, 27 objects/s recovering 2024-09-21T17:33:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:33:59 smithi157 ceph-mon[49460]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:33:59.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 236697 8.71 MB/sec execute 544 sec latency 68.166 ms 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 7 2024-09-21T17:33:59.826 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:33:59.827 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:33:59.827 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T17:33:59.827 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:33:59.827 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:33:59.827 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T17:33:59.827 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 11 2024-09-21T17:33:59.827 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:33:59.827 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:34:00.428 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:34:00.428 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:34:00.428 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:34:00.428 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "13/23 daemons upgraded", 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:34:00.429 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:34:00.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:00 smithi018 ceph-mon[63924]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:00.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:00 smithi018 ceph-mon[63924]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:00.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:00 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1442168755' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:00 smithi157 ceph-mon[49460]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:00 smithi157 ceph-mon[49460]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:00 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1442168755' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:00.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 236956 8.70 MB/sec execute 545 sec latency 49.373 ms 2024-09-21T17:34:01.169 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:34:01.229 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:01 smithi018 ceph-mon[63924]: pgmap v265: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.7 MiB/s wr, 440 op/s; 1.2 MiB/s, 27 keys/s, 18 objects/s recovering 2024-09-21T17:34:01.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:01.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:34:01.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:01 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1015545039' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:34:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:01 smithi157 ceph-mon[49460]: pgmap v265: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.7 MiB/s wr, 440 op/s; 1.2 MiB/s, 27 keys/s, 18 objects/s recovering 2024-09-21T17:34:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:34:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:01 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1015545039' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:34:01.789 INFO:teuthology.orchestra.run.smithi018.stdout: 5 237252 8.69 MB/sec execute 546 sec latency 43.979 ms 2024-09-21T17:34:02.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:02 smithi018 ceph-mon[63924]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:02.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:02 smithi157 ceph-mon[49460]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:02.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 237525 8.70 MB/sec execute 547 sec latency 57.681 ms 2024-09-21T17:34:03.230 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:03 smithi018 ceph-mon[63924]: pgmap v266: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.6 MiB/s wr, 471 op/s; 1.0 MiB/s, 23 keys/s, 15 objects/s recovering 2024-09-21T17:34:03.231 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T17:34:03.522 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:03 smithi157 ceph-mon[49460]: pgmap v266: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.6 MiB/s wr, 471 op/s; 1.0 MiB/s, 23 keys/s, 15 objects/s recovering 2024-09-21T17:34:03.522 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T17:34:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 237815 8.70 MB/sec execute 548 sec latency 50.326 ms 2024-09-21T17:34:04.287 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:04 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T17:34:04.288 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:04 smithi018 ceph-mon[63924]: Upgrade: osd.7 is safe to restart 2024-09-21T17:34:04.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:04 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T17:34:04.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:04 smithi157 ceph-mon[49460]: Upgrade: osd.7 is safe to restart 2024-09-21T17:34:04.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 238158 8.70 MB/sec execute 549 sec latency 65.221 ms 2024-09-21T17:34:05.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:05 smithi018 ceph-mon[63924]: pgmap v267: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 384 op/s; 759 KiB/s, 20 keys/s, 10 objects/s recovering 2024-09-21T17:34:05.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:05.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T17:34:05.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:05.518 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:05 smithi157 ceph-mon[49460]: pgmap v267: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 384 op/s; 759 KiB/s, 20 keys/s, 10 objects/s recovering 2024-09-21T17:34:05.518 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:05.518 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T17:34:05.518 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:05.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 238430 8.69 MB/sec execute 550 sec latency 67.049 ms 2024-09-21T17:34:06.248 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:06 smithi157 ceph-mon[49460]: Upgrade: Updating osd.7 2024-09-21T17:34:06.248 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:06 smithi157 ceph-mon[49460]: Deploying daemon osd.7 on smithi157 2024-09-21T17:34:06.248 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:06.382 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:06 smithi018 ceph-mon[63924]: Upgrade: Updating osd.7 2024-09-21T17:34:06.382 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:06 smithi018 ceph-mon[63924]: Deploying daemon osd.7 on smithi157 2024-09-21T17:34:06.382 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:06.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 238631 8.68 MB/sec execute 551 sec latency 67.253 ms 2024-09-21T17:34:07.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:07 smithi018 ceph-mon[63924]: pgmap v268: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 395 op/s; 595 KiB/s, 18 keys/s, 5 objects/s recovering 2024-09-21T17:34:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:07 smithi157 ceph-mon[49460]: pgmap v268: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 395 op/s; 595 KiB/s, 18 keys/s, 5 objects/s recovering 2024-09-21T17:34:07.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 238883 8.68 MB/sec execute 552 sec latency 45.057 ms 2024-09-21T17:34:08.634 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:08 smithi157 systemd[1]: Stopping Ceph osd.7 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:34:08.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 239134 8.68 MB/sec execute 553 sec latency 54.421 ms 2024-09-21T17:34:09.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:08 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[39009]: 2024-09-21T17:34:08.630+0000 7f3f57ab0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:34:09.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:08 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[39009]: 2024-09-21T17:34:08.630+0000 7f3f57ab0700 -1 osd.7 141 *** Got signal Terminated *** 2024-09-21T17:34:09.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:08 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[39009]: 2024-09-21T17:34:08.630+0000 7f3f57ab0700 -1 osd.7 141 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:34:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:09 smithi157 ceph-mon[49460]: pgmap v269: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 439 op/s; 182 KiB/s, 18 keys/s, 2 objects/s recovering 2024-09-21T17:34:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:09 smithi157 ceph-mon[49460]: osd.7 marked itself down and dead 2024-09-21T17:34:09.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:09 smithi018 ceph-mon[63924]: pgmap v269: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 439 op/s; 182 KiB/s, 18 keys/s, 2 objects/s recovering 2024-09-21T17:34:09.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:09 smithi018 ceph-mon[63924]: osd.7 marked itself down and dead 2024-09-21T17:34:09.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 239320 8.67 MB/sec execute 554 sec latency 556.374 ms 2024-09-21T17:34:10.241 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:09 smithi157 podman[58943]: 2024-09-21 17:34:09.924163931 +0000 UTC m=+1.428426165 container died b08f1b06fb678351ec7838d0ecca3a1aedc6e4eb121a588ee3e22da7c57738cc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, distribution-scope=public, GIT_BRANCH=HEAD, name=centos-stream, io.openshift.expose-services=, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, release=754) 2024-09-21T17:34:10.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:10 smithi157 podman[58943]: 2024-09-21 17:34:10.355030884 +0000 UTC m=+1.859293110 container cleanup b08f1b06fb678351ec7838d0ecca3a1aedc6e4eb121a588ee3e22da7c57738cc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, maintainer=Guillaume Abrioux , ceph=True, vendor=Red Hat, Inc., architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, RELEASE=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8) 2024-09-21T17:34:10.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:10 smithi157 bash[58943]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7 2024-09-21T17:34:10.529 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:10 smithi018 ceph-mon[63924]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:34:10.529 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:10 smithi018 ceph-mon[63924]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T17:34:10.529 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:10 smithi018 ceph-mon[63924]: osdmap e142: 8 total, 7 up, 8 in 2024-09-21T17:34:10.529 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:10 smithi018 ceph-mon[63924]: osdmap e143: 8 total, 7 up, 8 in 2024-09-21T17:34:10.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:10 smithi157 ceph-mon[49460]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T17:34:10.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:10 smithi157 ceph-mon[49460]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T17:34:10.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:10 smithi157 ceph-mon[49460]: osdmap e142: 8 total, 7 up, 8 in 2024-09-21T17:34:10.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:10 smithi157 ceph-mon[49460]: osdmap e143: 8 total, 7 up, 8 in 2024-09-21T17:34:10.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 239506 8.67 MB/sec execute 555 sec latency 1099.973 ms 2024-09-21T17:34:11.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:10 smithi157 podman[58961]: 2024-09-21 17:34:10.623854526 +0000 UTC m=+0.695978892 container remove b08f1b06fb678351ec7838d0ecca3a1aedc6e4eb121a588ee3e22da7c57738cc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, version=8, vcs-type=git, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, distribution-scope=public) 2024-09-21T17:34:11.287 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:11 smithi018 ceph-mon[63924]: pgmap v271: 97 pgs: 12 stale+active+clean, 85 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.9 MiB/s wr, 360 op/s 2024-09-21T17:34:11.371 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:11 smithi157 ceph-mon[49460]: pgmap v271: 97 pgs: 12 stale+active+clean, 85 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.9 MiB/s wr, 360 op/s 2024-09-21T17:34:11.371 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:11 smithi157 podman[59050]: 2024-09-21 17:34:10.987073909 +0000 UTC m=+0.016286836 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:34:11.371 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:11 smithi157 podman[59050]: 2024-09-21 17:34:11.099749924 +0000 UTC m=+0.128962832 container create d93fe8e23ee47ce73b4e8a9305a14c1fbb2a624f601638ca9056b9c0d65b35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T17:34:11.371 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:11 smithi157 podman[59050]: 2024-09-21 17:34:11.369547763 +0000 UTC m=+0.398760687 container init d93fe8e23ee47ce73b4e8a9305a14c1fbb2a624f601638ca9056b9c0d65b35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T17:34:11.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:11 smithi157 podman[59050]: 2024-09-21 17:34:11.373504718 +0000 UTC m=+0.402717626 container start d93fe8e23ee47ce73b4e8a9305a14c1fbb2a624f601638ca9056b9c0d65b35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T17:34:11.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:11 smithi157 podman[59050]: 2024-09-21 17:34:11.486285584 +0000 UTC m=+0.515498504 container attach d93fe8e23ee47ce73b4e8a9305a14c1fbb2a624f601638ca9056b9c0d65b35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T17:34:11.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:11 smithi157 podman[59050]: 2024-09-21 17:34:11.569682923 +0000 UTC m=+0.598895838 container died d93fe8e23ee47ce73b4e8a9305a14c1fbb2a624f601638ca9056b9c0d65b35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T17:34:11.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 239886 8.67 MB/sec execute 556 sec latency 35.433 ms 2024-09-21T17:34:12.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:12 smithi018 ceph-mon[63924]: Health check failed: Degraded data redundancy: 181/1536 objects degraded (11.784%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:12.651 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:12 smithi157 ceph-mon[49460]: Health check failed: Degraded data redundancy: 181/1536 objects degraded (11.784%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:12.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 240198 8.67 MB/sec execute 557 sec latency 35.884 ms 2024-09-21T17:34:13.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:12 smithi157 podman[59074]: 2024-09-21 17:34:12.649689564 +0000 UTC m=+1.071207504 container remove d93fe8e23ee47ce73b4e8a9305a14c1fbb2a624f601638ca9056b9c0d65b35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-21T17:34:13.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:12 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-21T17:34:13.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:12 smithi157 systemd[1]: Stopped Ceph osd.7 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:34:13.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:12 smithi157 systemd[1]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.7.service: Consumed 5min 46.381s CPU time. 2024-09-21T17:34:13.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:12 smithi157 systemd[1]: Starting Ceph osd.7 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:34:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:13 smithi157 ceph-mon[49460]: pgmap v273: 97 pgs: 5 active+undersized, 2 stale+active+clean, 23 active+undersized+degraded, 67 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s wr, 459 op/s; 181/1536 objects degraded (11.784%); 327 B/s, 0 objects/s recovering 2024-09-21T17:34:13.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 podman[59174]: 2024-09-21 17:34:12.991669266 +0000 UTC m=+0.015903788 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:34:13.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 podman[59174]: 2024-09-21 17:34:13.126327617 +0000 UTC m=+0.150562121 container create 71582c2944f8fc4519256e32dc7b96f1146baa199f671ef8869804f32bae28b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:34:13.580 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:13 smithi018 ceph-mon[63924]: pgmap v273: 97 pgs: 5 active+undersized, 2 stale+active+clean, 23 active+undersized+degraded, 67 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s wr, 459 op/s; 181/1536 objects degraded (11.784%); 327 B/s, 0 objects/s recovering 2024-09-21T17:34:13.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 240516 8.67 MB/sec execute 558 sec latency 52.326 ms 2024-09-21T17:34:14.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 podman[59174]: 2024-09-21 17:34:13.755252627 +0000 UTC m=+0.779487151 container init 71582c2944f8fc4519256e32dc7b96f1146baa199f671ef8869804f32bae28b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T17:34:14.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 podman[59174]: 2024-09-21 17:34:13.758641675 +0000 UTC m=+0.782876182 container start 71582c2944f8fc4519256e32dc7b96f1146baa199f671ef8869804f32bae28b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T17:34:14.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 podman[59174]: 2024-09-21 17:34:13.863286872 +0000 UTC m=+0.887521381 container attach 71582c2944f8fc4519256e32dc7b96f1146baa199f671ef8869804f32bae28b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T17:34:14.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:34:14.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 bash[59174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:34:14.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:34:14.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:13 smithi157 bash[59174]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T17:34:14.599 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:14 smithi018 ceph-mon[63924]: pgmap v274: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 414 op/s; 252/1575 objects degraded (16.000%); 327 B/s, 0 objects/s recovering 2024-09-21T17:34:14.690 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:14 smithi157 ceph-mon[49460]: pgmap v274: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 414 op/s; 252/1575 objects degraded (16.000%); 327 B/s, 0 objects/s recovering 2024-09-21T17:34:14.690 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T17:34:14.690 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 bash[59174]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T17:34:14.690 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T17:34:14.690 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 bash[59174]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T17:34:14.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 240780 8.66 MB/sec execute 559 sec latency 47.444 ms 2024-09-21T17:34:15.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T17:34:15.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 bash[59174]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T17:34:15.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T17:34:15.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 bash[59174]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T17:34:15.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T17:34:15.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 bash[59174]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T17:34:15.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T17:34:15.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 bash[59174]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T17:34:15.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate[59188]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-21T17:34:15.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 bash[59174]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-21T17:34:15.029 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:14 smithi157 podman[59174]: 2024-09-21 17:34:14.718652254 +0000 UTC m=+1.742886777 container died 71582c2944f8fc4519256e32dc7b96f1146baa199f671ef8869804f32bae28b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:34:15.679 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:15 smithi157 podman[59328]: 2024-09-21 17:34:15.418265185 +0000 UTC m=+0.690669671 container remove 71582c2944f8fc4519256e32dc7b96f1146baa199f671ef8869804f32bae28b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-activate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2) 2024-09-21T17:34:15.679 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:15 smithi157 podman[59385]: 2024-09-21 17:34:15.538751597 +0000 UTC m=+0.017328459 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:34:15.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 241103 8.67 MB/sec execute 560 sec latency 60.161 ms 2024-09-21T17:34:15.957 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:15 smithi157 podman[59385]: 2024-09-21 17:34:15.677628095 +0000 UTC m=+0.156204929 container create 85bf07d224b1d30fa641ebf8d944196a3f589a38f717cddab35c02277d912848 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T17:34:15.957 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:15 smithi157 podman[59385]: 2024-09-21 17:34:15.955560173 +0000 UTC m=+0.434137015 container init 85bf07d224b1d30fa641ebf8d944196a3f589a38f717cddab35c02277d912848 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:34:16.278 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:15 smithi157 podman[59385]: 2024-09-21 17:34:15.959493076 +0000 UTC m=+0.438069910 container start 85bf07d224b1d30fa641ebf8d944196a3f589a38f717cddab35c02277d912848 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T17:34:16.279 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:15 smithi157 ceph-osd[59404]: -- 172.21.15.157:0/778095039 <== mon.1 v2:172.21.15.157:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c9804481e0 con 0x55c97f5e7c00 2024-09-21T17:34:16.279 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:16 smithi157 bash[59385]: 85bf07d224b1d30fa641ebf8d944196a3f589a38f717cddab35c02277d912848 2024-09-21T17:34:16.279 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:16 smithi157 systemd[1]: Started Ceph osd.7 for ec44ef70-783c-11ef-baf6-efdc52797490. 2024-09-21T17:34:16.638 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:16.638 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:34:16.638 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:16 smithi018 ceph-mon[63924]: pgmap v275: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 333 op/s; 255/1614 objects degraded (15.799%); 327 B/s, 0 objects/s recovering 2024-09-21T17:34:16.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:16.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:16.639 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:34:16.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 241417 8.67 MB/sec execute 561 sec latency 63.037 ms 2024-09-21T17:34:16.904 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:16.904 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:34:16.904 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:16 smithi157 ceph-mon[49460]: pgmap v275: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 333 op/s; 255/1614 objects degraded (15.799%); 327 B/s, 0 objects/s recovering 2024-09-21T17:34:16.904 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:16.904 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:16.904 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:34:17.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 241911 8.67 MB/sec execute 562 sec latency 45.562 ms 2024-09-21T17:34:17.924 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:17 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:34:17.602+0000 7f3617cc7640 -1 Falling back to public interface 2024-09-21T17:34:18.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:18 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 255/1608 objects degraded (15.858%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:18.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:18 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 255/1608 objects degraded (15.858%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:18.800 INFO:teuthology.orchestra.run.smithi018.stdout: 5 242264 8.67 MB/sec execute 563 sec latency 42.817 ms 2024-09-21T17:34:19.416 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:19 smithi018 ceph-mon[63924]: pgmap v276: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s wr, 362 op/s; 255/1608 objects degraded (15.858%); 296 B/s, 0 objects/s recovering 2024-09-21T17:34:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:19 smithi157 ceph-mon[49460]: pgmap v276: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s wr, 362 op/s; 255/1608 objects degraded (15.858%); 296 B/s, 0 objects/s recovering 2024-09-21T17:34:19.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 242658 8.68 MB/sec execute 564 sec latency 45.426 ms 2024-09-21T17:34:20.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 243098 8.69 MB/sec execute 565 sec latency 38.435 ms 2024-09-21T17:34:21.194 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:21 smithi018 ceph-mon[63924]: pgmap v277: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 502 op/s; 249/1584 objects degraded (15.720%); 261 B/s, 0 objects/s recovering 2024-09-21T17:34:21.194 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:21.195 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:21.195 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:21.195 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:21 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:21.447 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:21 smithi157 ceph-mon[49460]: pgmap v277: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 502 op/s; 249/1584 objects degraded (15.720%); 261 B/s, 0 objects/s recovering 2024-09-21T17:34:21.448 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:21.448 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:21.448 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:21.448 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:21 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 243461 8.69 MB/sec execute 566 sec latency 46.726 ms 2024-09-21T17:34:22.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 243763 8.70 MB/sec execute 567 sec latency 48.805 ms 2024-09-21T17:34:23.212 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:23 smithi018 ceph-mon[63924]: pgmap v278: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.2 MiB/s wr, 488 op/s; 255/1647 objects degraded (15.483%); 221 B/s, 0 objects/s recovering 2024-09-21T17:34:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:23 smithi157 ceph-mon[49460]: pgmap v278: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.2 MiB/s wr, 488 op/s; 255/1647 objects degraded (15.483%); 221 B/s, 0 objects/s recovering 2024-09-21T17:34:23.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 244080 8.69 MB/sec execute 568 sec latency 55.847 ms 2024-09-21T17:34:24.356 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:24 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 253/1650 objects degraded (15.333%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:24.472 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:24 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 253/1650 objects degraded (15.333%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:24.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:24 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:34:24.353+0000 7f3617cc7640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-21T17:34:24.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 244338 8.68 MB/sec execute 569 sec latency 64.947 ms 2024-09-21T17:34:25.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: pgmap v279: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 510 op/s; 253/1650 objects degraded (15.333%) 2024-09-21T17:34:25.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:25.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:34:25.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.252 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T17:34:25.253 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T17:34:25.254 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T17:34:25.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: pgmap v279: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 510 op/s; 253/1650 objects degraded (15.333%) 2024-09-21T17:34:25.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:25.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:34:25.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T17:34:25.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T17:34:25.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T17:34:25.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 244582 8.68 MB/sec execute 570 sec latency 49.726 ms 2024-09-21T17:34:26.278 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:25 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:34:25.998+0000 7f3617cc7640 -1 osd.7 141 log_to_monitors true 2024-09-21T17:34:26.278 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:34:26.182+0000 7f360fa71640 -1 osd.7 141 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T17:34:26.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all osd 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: from='osd.7 [v2:172.21.15.157:6824/3906450478,v1:172.21.15.157:6825/3906450478]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: from='osd.7 [v2:172.21.15.157:6824/3906450478,v1:172.21.15.157:6825/3906450478]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:34:26.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:26 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi018.wzitfj"]} : dispatch 2024-09-21T17:34:26.317 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all osd 2024-09-21T17:34:26.317 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T17:34:26.317 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: from='osd.7 [v2:172.21.15.157:6824/3906450478,v1:172.21.15.157:6825/3906450478]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T17:34:26.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T17:34:26.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T17:34:26.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T17:34:26.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T17:34:26.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T17:34:26.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: from='osd.7 [v2:172.21.15.157:6824/3906450478,v1:172.21.15.157:6825/3906450478]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:34:26.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi157", "root=default"]} : dispatch 2024-09-21T17:34:26.318 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:26 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi018.wzitfj"]} : dispatch 2024-09-21T17:34:26.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 244915 8.68 MB/sec execute 571 sec latency 55.599 ms 2024-09-21T17:34:27.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:27 smithi018 ceph-mon[63924]: pgmap v280: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 248/1602 objects degraded (15.481%) 2024-09-21T17:34:27.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:27 smithi018 ceph-mon[63924]: Upgrade: It appears safe to stop mds.foofs.smithi018.wzitfj 2024-09-21T17:34:27.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:27 smithi018 ceph-mon[63924]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:34:27.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:27 smithi018 ceph-mon[63924]: osd.7 [v2:172.21.15.157:6824/3906450478,v1:172.21.15.157:6825/3906450478] boot 2024-09-21T17:34:27.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:27 smithi018 ceph-mon[63924]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T17:34:27.406 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:27 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T17:34:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:27 smithi157 ceph-mon[49460]: pgmap v280: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 248/1602 objects degraded (15.481%) 2024-09-21T17:34:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:27 smithi157 ceph-mon[49460]: Upgrade: It appears safe to stop mds.foofs.smithi018.wzitfj 2024-09-21T17:34:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:27 smithi157 ceph-mon[49460]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T17:34:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:27 smithi157 ceph-mon[49460]: osd.7 [v2:172.21.15.157:6824/3906450478,v1:172.21.15.157:6825/3906450478] boot 2024-09-21T17:34:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:27 smithi157 ceph-mon[49460]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T17:34:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:27 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T17:34:27.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245129 8.68 MB/sec execute 572 sec latency 643.809 ms 2024-09-21T17:34:28.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245129 8.66 MB/sec execute 573 sec latency 1643.839 ms 2024-09-21T17:34:29.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:29 smithi018 ceph-mon[63924]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.1 MiB/s wr, 459 op/s; 245/1551 objects degraded (15.796%) 2024-09-21T17:34:29.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:29 smithi018 ceph-mon[63924]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T17:34:29.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:29 smithi018 ceph-mon[63924]: Upgrade: Updating mds.foofs.smithi018.wzitfj 2024-09-21T17:34:29.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:29 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:29.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:29 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi018.wzitfj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T17:34:29.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:29 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:29.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:29 smithi157 ceph-mon[49460]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.1 MiB/s wr, 459 op/s; 245/1551 objects degraded (15.796%) 2024-09-21T17:34:29.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:29 smithi157 ceph-mon[49460]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T17:34:29.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:29 smithi157 ceph-mon[49460]: Upgrade: Updating mds.foofs.smithi018.wzitfj 2024-09-21T17:34:29.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:29 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:29.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:29 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi018.wzitfj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T17:34:29.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:29 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:29.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245129 8.65 MB/sec execute 574 sec latency 2643.869 ms 2024-09-21T17:34:30.389 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:30 smithi018 ceph-mon[63924]: Deploying daemon mds.foofs.smithi018.wzitfj on smithi018 2024-09-21T17:34:30.389 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:30 smithi018 ceph-mon[63924]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T17:34:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:30 smithi157 ceph-mon[49460]: Deploying daemon mds.foofs.smithi018.wzitfj on smithi018 2024-09-21T17:34:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:30 smithi157 ceph-mon[49460]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T17:34:30.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:30 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[63920]: 2024-09-21T17:34:30.574+0000 7f91b084c640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:34:30.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245300 8.64 MB/sec execute 575 sec latency 2980.214 ms 2024-09-21T17:34:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: pgmap v286: 97 pgs: 6 remapped+peering, 1 active+recovery_wait+degraded, 5 active+undersized, 23 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 MiB/s wr, 346 op/s; 190/1533 objects degraded (12.394%) 2024-09-21T17:34:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 190/1533 objects degraded (12.394%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: Standby daemon mds.foofs.smithi157.iijaxo assigned to filesystem foofs as rank 0 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: fsmap foofs:1/1 {0=foofs.smithi157.iijaxo=up:replay} 2024-09-21T17:34:31.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"} : dispatch 2024-09-21T17:34:31.562 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: pgmap v286: 97 pgs: 6 remapped+peering, 1 active+recovery_wait+degraded, 5 active+undersized, 23 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 MiB/s wr, 346 op/s; 190/1533 objects degraded (12.394%) 2024-09-21T17:34:31.562 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 190/1533 objects degraded (12.394%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:31.562 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:31.562 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: Standby daemon mds.foofs.smithi157.iijaxo assigned to filesystem foofs as rank 0 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: fsmap foofs:1/1 {0=foofs.smithi157.iijaxo=up:replay} 2024-09-21T17:34:31.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"} : dispatch 2024-09-21T17:34:31.563 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:34:31.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245300 8.62 MB/sec execute 576 sec latency 1237.422 ms 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (9m) 2m ago 18m 19.7M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (5m) 2m ago 18m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (5m) 12s ago 17m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (9m) 2m ago 18m 47.5M - dad864ee21e9 48d78bd31599 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (12m) 2m ago 12m 38.4M - 17.2.0 e1d6a67b021e 86858a6f72d1 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (12m) 12s ago 12m 16.8M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (8m) 2m ago 19m 520M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (7m) 12s ago 17m 459M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:34:32.304 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (7m) 2m ago 19m 51.6M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (6m) 12s ago 17m 45.9M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (11m) 2m ago 11m 276M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (11m) 12s ago 11m 87.4M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (8m) 2m ago 18m 10.1M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (8m) 12s ago 17m 11.7M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (5m) 2m ago 16m 337M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (4m) 2m ago 16m 272M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (3m) 2m ago 15m 218M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (2m) 2m ago 15m 13.3M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (2m) 12s ago 15m 333M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 2552dce6837e 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (92s) 12s ago 14m 272M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d abad654e4600 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (54s) 12s ago 14m 238M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d fd2e6eb21352 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (16s) 12s ago 13m 16.5M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 85bf07d224b1 2024-09-21T17:34:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (7m) 2m ago 17m 53.1M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:34:32.767 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:34:32.767 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:34:32.767 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:34:32.767 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:34:32.767 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:34:32.767 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:34:32.767 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:34:32.767 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 8 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 12 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:34:32.768 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:34:32.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245300 8.61 MB/sec execute 577 sec latency 2237.453 ms 2024-09-21T17:34:32.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:32 smithi018 ceph-mon[63924]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:32.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:32 smithi018 ceph-mon[63924]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T17:34:32.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:32 smithi018 ceph-mon[63924]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:32.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:32 smithi018 ceph-mon[63924]: pgmap v289: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 peering, 6 remapped+peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded, 83 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.7 MiB/s wr, 347 op/s; 72/1530 objects degraded (4.706%); 2.0 MiB/s, 32 objects/s recovering 2024-09-21T17:34:32.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:32 smithi018 ceph-mon[63924]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:32 smithi157 ceph-mon[49460]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:32 smithi157 ceph-mon[49460]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T17:34:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:32 smithi157 ceph-mon[49460]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:32 smithi157 ceph-mon[49460]: pgmap v289: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 peering, 6 remapped+peering, 1 active+recovery_wait+degraded, 2 active+undersized+degraded, 83 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.7 MiB/s wr, 347 op/s; 72/1530 objects degraded (4.706%); 2.0 MiB/s, 32 objects/s recovering 2024-09-21T17:34:33.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:32 smithi157 ceph-mon[49460]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:33.150 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "osd", 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "14/23 daemons upgraded", 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:34:33.151 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN 1 filesystem is degraded; insufficient standby MDS daemons available; Degraded data redundancy: 72/1530 objects degraded (4.706%), 4 pgs degraded 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout: fs foofs is degraded 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout: have 0; want 1 more 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 72/1530 objects degraded (4.706%), 4 pgs degraded 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.7 is active+recovery_wait+degraded, acting [3,7,0] 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-09-21T17:34:33.595 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-09-21T17:34:33.596 INFO:teuthology.orchestra.run.smithi018.stdout: pg 3.16 is active+recovery_wait+undersized+degraded+remapped, acting [5,1] 2024-09-21T17:34:33.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245300 8.59 MB/sec execute 578 sec latency 3237.507 ms 2024-09-21T17:34:33.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:33 smithi018 ceph-mon[63924]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T17:34:33.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:33 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2030422890' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:33.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:33 smithi018 ceph-mon[63924]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:33 smithi157 ceph-mon[49460]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T17:34:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:33 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2030422890' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:34.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:33 smithi157 ceph-mon[49460]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:34:34.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245300 8.58 MB/sec execute 579 sec latency 4237.557 ms 2024-09-21T17:34:34.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:34 smithi018 ceph-mon[63924]: reconnect by client.24347 172.21.15.157:0/265609017 after 0.000999981 2024-09-21T17:34:34.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:34 smithi018 ceph-mon[63924]: reconnect by client.14554 172.21.15.18:0/358269258 after 0.000999981 2024-09-21T17:34:34.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:34 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:reconnect 2024-09-21T17:34:34.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:34 smithi018 ceph-mon[63924]: fsmap foofs:1/1 {0=foofs.smithi157.iijaxo=up:reconnect} 2024-09-21T17:34:34.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:34 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1620127119' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:34:34.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:34 smithi018 ceph-mon[63924]: pgmap v291: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 peering, 2 remapped+peering, 2 active+undersized+degraded, 88 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 693 KiB/s wr, 108 op/s; 54/1536 objects degraded (3.516%); 4.9 MiB/s, 35 objects/s recovering 2024-09-21T17:34:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:34 smithi157 ceph-mon[49460]: reconnect by client.24347 172.21.15.157:0/265609017 after 0.000999981 2024-09-21T17:34:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:34 smithi157 ceph-mon[49460]: reconnect by client.14554 172.21.15.18:0/358269258 after 0.000999981 2024-09-21T17:34:35.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:34 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:reconnect 2024-09-21T17:34:35.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:34 smithi157 ceph-mon[49460]: fsmap foofs:1/1 {0=foofs.smithi157.iijaxo=up:reconnect} 2024-09-21T17:34:35.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:34 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1620127119' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:34:35.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:34 smithi157 ceph-mon[49460]: pgmap v291: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 peering, 2 remapped+peering, 2 active+undersized+degraded, 88 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 693 KiB/s wr, 108 op/s; 54/1536 objects degraded (3.516%); 4.9 MiB/s, 35 objects/s recovering 2024-09-21T17:34:35.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245300 8.56 MB/sec execute 580 sec latency 5237.608 ms 2024-09-21T17:34:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:35 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:rejoin 2024-09-21T17:34:35.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:35 smithi018 ceph-mon[63924]: fsmap foofs:1/1 {0=foofs.smithi157.iijaxo=up:rejoin} 2024-09-21T17:34:36.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:35 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:rejoin 2024-09-21T17:34:36.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:35 smithi157 ceph-mon[49460]: fsmap foofs:1/1 {0=foofs.smithi157.iijaxo=up:rejoin} 2024-09-21T17:34:36.623 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:36 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:clientreplay 2024-09-21T17:34:36.623 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:36 smithi018 ceph-mon[63924]: fsmap foofs:1/1 {0=foofs.smithi157.iijaxo=up:clientreplay} 2024-09-21T17:34:36.624 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:36 smithi018 ceph-mon[63924]: daemon mds.foofs.smithi157.iijaxo is now active in filesystem foofs as rank 0 2024-09-21T17:34:36.624 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:36 smithi018 ceph-mon[63924]: pgmap v292: 97 pgs: 1 active+undersized+remapped, 2 peering, 2 remapped+peering, 2 active+undersized+degraded, 90 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 568 KiB/s wr, 96 op/s; 29/1497 objects degraded (1.937%); 4.0 MiB/s, 33 objects/s recovering 2024-09-21T17:34:36.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245372 8.55 MB/sec execute 581 sec latency 6055.527 ms 2024-09-21T17:34:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:36 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:clientreplay 2024-09-21T17:34:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:36 smithi157 ceph-mon[49460]: fsmap foofs:1/1 {0=foofs.smithi157.iijaxo=up:clientreplay} 2024-09-21T17:34:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:36 smithi157 ceph-mon[49460]: daemon mds.foofs.smithi157.iijaxo is now active in filesystem foofs as rank 0 2024-09-21T17:34:37.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:36 smithi157 ceph-mon[49460]: pgmap v292: 97 pgs: 1 active+undersized+remapped, 2 peering, 2 remapped+peering, 2 active+undersized+degraded, 90 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 568 KiB/s wr, 96 op/s; 29/1497 objects degraded (1.937%); 4.0 MiB/s, 33 objects/s recovering 2024-09-21T17:34:37.608 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:37 smithi018 ceph-mon[63924]: Health check update: Degraded data redundancy: 29/1497 objects degraded (1.937%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:37.608 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:37 smithi018 ceph-mon[63924]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T17:34:37.608 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:37 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:active 2024-09-21T17:34:37.608 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:37 smithi018 ceph-mon[63924]: fsmap foofs:1 {0=foofs.smithi157.iijaxo=up:active} 2024-09-21T17:34:37.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 245757 8.55 MB/sec execute 582 sec latency 39.639 ms 2024-09-21T17:34:38.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:37 smithi157 ceph-mon[49460]: Health check update: Degraded data redundancy: 29/1497 objects degraded (1.937%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T17:34:38.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:37 smithi157 ceph-mon[49460]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T17:34:38.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:37 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.157:6832/1527840175,v1:172.21.15.157:6833/1527840175] up:active 2024-09-21T17:34:38.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:37 smithi157 ceph-mon[49460]: fsmap foofs:1 {0=foofs.smithi157.iijaxo=up:active} 2024-09-21T17:34:38.648 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:38 smithi018 ceph-mon[63924]: pgmap v293: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 486 KiB/s wr, 88 op/s; 3.5 MiB/s, 39 objects/s recovering 2024-09-21T17:34:38.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 246092 8.55 MB/sec execute 583 sec latency 59.434 ms 2024-09-21T17:34:39.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:38 smithi157 ceph-mon[49460]: pgmap v293: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 486 KiB/s wr, 88 op/s; 3.5 MiB/s, 39 objects/s recovering 2024-09-21T17:34:39.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:39 smithi018 ceph-mon[63924]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1497 objects degraded (1.937%), 2 pgs degraded) 2024-09-21T17:34:39.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 246550 8.56 MB/sec execute 584 sec latency 42.061 ms 2024-09-21T17:34:40.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:39 smithi157 ceph-mon[49460]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1497 objects degraded (1.937%), 2 pgs degraded) 2024-09-21T17:34:40.802 INFO:teuthology.orchestra.run.smithi018.stdout: 5 246986 8.57 MB/sec execute 585 sec latency 61.498 ms 2024-09-21T17:34:40.882 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:40 smithi018 ceph-mon[63924]: pgmap v294: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 424 KiB/s wr, 74 op/s; 2.6 MiB/s, 24 objects/s recovering 2024-09-21T17:34:40.983 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:40 smithi157 ceph-mon[49460]: pgmap v294: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 424 KiB/s wr, 74 op/s; 2.6 MiB/s, 24 objects/s recovering 2024-09-21T17:34:41.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 247212 8.57 MB/sec execute 586 sec latency 68.048 ms 2024-09-21T17:34:42.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:42.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:42.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:34:42.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:41 smithi018 ceph-mon[63924]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T17:34:42.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:41 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:34:42.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:41 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:boot 2024-09-21T17:34:42.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:41 smithi018 ceph-mon[63924]: fsmap foofs:1 {0=foofs.smithi157.iijaxo=up:active} 1 up:standby 2024-09-21T17:34:42.169 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:41 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"} : dispatch 2024-09-21T17:34:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:41 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:41 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:41 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:34:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:41 smithi157 ceph-mon[49460]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T17:34:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:41 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:34:42.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:41 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:boot 2024-09-21T17:34:42.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:41 smithi157 ceph-mon[49460]: fsmap foofs:1 {0=foofs.smithi157.iijaxo=up:active} 1 up:standby 2024-09-21T17:34:42.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:41 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi018.wzitfj"} : dispatch 2024-09-21T17:34:42.801 INFO:teuthology.orchestra.run.smithi018.stdout: 5 247492 8.57 MB/sec execute 587 sec latency 68.025 ms 2024-09-21T17:34:43.191 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:42 smithi018 ceph-mon[63924]: pgmap v295: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.1 MiB/s wr, 185 op/s; 1.8 MiB/s, 16 objects/s recovering 2024-09-21T17:34:43.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:42 smithi157 ceph-mon[49460]: pgmap v295: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.1 MiB/s wr, 185 op/s; 1.8 MiB/s, 16 objects/s recovering 2024-09-21T17:34:43.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 247757 8.56 MB/sec execute 588 sec latency 87.247 ms 2024-09-21T17:34:44.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 247950 8.55 MB/sec execute 589 sec latency 68.400 ms 2024-09-21T17:34:45.250 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:45 smithi018 ceph-mon[63924]: pgmap v296: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.7 MiB/s wr, 306 op/s; 1.6 MiB/s, 14 objects/s recovering 2024-09-21T17:34:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:45 smithi157 ceph-mon[49460]: pgmap v296: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.7 MiB/s wr, 306 op/s; 1.6 MiB/s, 14 objects/s recovering 2024-09-21T17:34:45.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 248180 8.55 MB/sec execute 590 sec latency 54.802 ms 2024-09-21T17:34:46.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:46.772 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:34:46.773 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:46.773 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:46 smithi018 ceph-mon[63924]: pgmap v297: 97 pgs: 97 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.9 MiB/s wr, 307 op/s; 160 KiB/s, 9 objects/s recovering 2024-09-21T17:34:46.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 248388 8.55 MB/sec execute 591 sec latency 49.478 ms 2024-09-21T17:34:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:34:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:47.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:46 smithi157 ceph-mon[49460]: pgmap v297: 97 pgs: 97 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.9 MiB/s wr, 307 op/s; 160 KiB/s, 9 objects/s recovering 2024-09-21T17:34:47.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 248656 8.55 MB/sec execute 592 sec latency 66.772 ms 2024-09-21T17:34:48.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 248829 8.54 MB/sec execute 593 sec latency 161.609 ms 2024-09-21T17:34:49.353 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:49 smithi018 ceph-mon[63924]: pgmap v298: 97 pgs: 97 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 3.6 MiB/s wr, 330 op/s; 150 KiB/s, 7 objects/s recovering 2024-09-21T17:34:49.353 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:49 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:49.353 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:49 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:49 smithi157 ceph-mon[49460]: pgmap v298: 97 pgs: 97 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 3.6 MiB/s wr, 330 op/s; 150 KiB/s, 7 objects/s recovering 2024-09-21T17:34:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:49 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:49 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:49.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 249002 8.53 MB/sec execute 594 sec latency 108.283 ms 2024-09-21T17:34:50.623 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:50 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:50.623 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:50 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:50.624 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:50 smithi018 ceph-mon[63924]: pgmap v299: 97 pgs: 97 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 4.2 MiB/s wr, 437 op/s 2024-09-21T17:34:50.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:50 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:50.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:50 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:50.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:50 smithi157 ceph-mon[49460]: pgmap v299: 97 pgs: 97 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 4.2 MiB/s wr, 437 op/s 2024-09-21T17:34:50.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 249182 8.52 MB/sec execute 595 sec latency 123.169 ms 2024-09-21T17:34:51.806 INFO:teuthology.orchestra.run.smithi018.stdout: 5 249356 8.51 MB/sec execute 596 sec latency 123.079 ms 2024-09-21T17:34:52.807 INFO:teuthology.orchestra.run.smithi018.stdout: 5 249937 8.53 MB/sec execute 597 sec latency 33.929 ms 2024-09-21T17:34:53.179 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:53 smithi018 ceph-mon[63924]: pgmap v300: 97 pgs: 97 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.4 MiB/s wr, 434 op/s 2024-09-21T17:34:53.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:53 smithi157 ceph-mon[49460]: pgmap v300: 97 pgs: 97 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.4 MiB/s wr, 434 op/s 2024-09-21T17:34:53.807 INFO:teuthology.orchestra.run.smithi018.stdout: 5 250493 8.55 MB/sec execute 598 sec latency 26.829 ms 2024-09-21T17:34:54.807 INFO:teuthology.orchestra.run.smithi018.stdout: 5 250771 8.54 MB/sec execute 599 sec latency 47.539 ms 2024-09-21T17:34:55.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:55 smithi018 ceph-mon[63924]: pgmap v301: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 3.8 MiB/s wr, 472 op/s 2024-09-21T17:34:55.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:55 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:55.203 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:55 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:55.204 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:55 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:55.204 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:55 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:34:55.204 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:55 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:55.204 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:55 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:55 smithi157 ceph-mon[49460]: pgmap v301: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 3.8 MiB/s wr, 472 op/s 2024-09-21T17:34:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:55 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:55 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:55 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:55.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:55 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:34:55.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:55 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:55.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:55 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:55.807 INFO:teuthology.orchestra.run.smithi018.stdout: 5 cleanup 600 sec 2024-09-21T17:34:56.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:56 smithi018 ceph-mon[63924]: Detected new or changed devices on smithi018 2024-09-21T17:34:56.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:56 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:34:56.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:56 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:56.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:56 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:56.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:56 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:56.216 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:56 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:56.217 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:56 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi157.iijaxo"]} : dispatch 2024-09-21T17:34:56.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:56 smithi157 ceph-mon[49460]: Detected new or changed devices on smithi018 2024-09-21T17:34:56.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:56 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:34:56.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:56 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:56.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:56 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:56.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:56 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:56.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:56 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:34:56.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:56 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi157.iijaxo"]} : dispatch 2024-09-21T17:34:56.530 INFO:teuthology.orchestra.run.smithi018.stdout: 0 cleanup 601 sec 2024-09-21T17:34:56.530 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:34:56.530 INFO:teuthology.orchestra.run.smithi018.stdout: Operation Count AvgLat MaxLat 2024-09-21T17:34:56.530 INFO:teuthology.orchestra.run.smithi018.stdout: ---------------------------------------- 2024-09-21T17:34:56.530 INFO:teuthology.orchestra.run.smithi018.stdout: NTCreateX 164857 5.187 6055.523 2024-09-21T17:34:56.530 INFO:teuthology.orchestra.run.smithi018.stdout: Close 120961 0.139 29.678 2024-09-21T17:34:56.530 INFO:teuthology.orchestra.run.smithi018.stdout: Rename 6970 10.439 2347.744 2024-09-21T17:34:56.530 INFO:teuthology.orchestra.run.smithi018.stdout: Unlink 33334 7.674 2978.795 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: Qpathinfo 149435 1.378 6040.746 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: Qfileinfo 26023 0.007 2.222 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: Qfsinfo 27346 0.647 55.433 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: Sfileinfo 13423 14.704 6044.116 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: Find 57662 1.001 100.004 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: WriteX 81298 16.169 6042.534 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: ReadX 257950 0.010 2.939 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: LockX 536 0.386 2.629 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: UnlockX 536 0.342 2.367 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: Flush 11535 0.003 0.167 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:34:56.531 INFO:teuthology.orchestra.run.smithi018.stdout:Throughput 8.54406 MB/sec 5 clients 5 procs max_latency=6055.527 ms 2024-09-21T17:34:56.533 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-21T17:34:56.570 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-21T17:34:56.713 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-21T17:34:56.739 INFO:teuthology.orchestra.run.smithi018.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-21T17:34:56.739 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:34:56.739 INFO:teuthology.orchestra.run.smithi018.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-21T17:34:56.739 INFO:teuthology.orchestra.run.smithi018.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-21T17:34:56.840 INFO:teuthology.orchestra.run.smithi018.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-21T17:34:56.840 INFO:teuthology.orchestra.run.smithi018.stdout:releasing clients 2024-09-21T17:34:57.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:57 smithi018 ceph-mon[63924]: Upgrade: It appears safe to stop mds.foofs.smithi157.iijaxo 2024-09-21T17:34:57.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:57 smithi018 ceph-mon[63924]: pgmap v302: 97 pgs: 97 active+clean; 347 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.2 MiB/s wr, 353 op/s 2024-09-21T17:34:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:57 smithi157 ceph-mon[49460]: Upgrade: It appears safe to stop mds.foofs.smithi157.iijaxo 2024-09-21T17:34:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:57 smithi157 ceph-mon[49460]: pgmap v302: 97 pgs: 97 active+clean; 347 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.2 MiB/s wr, 353 op/s 2024-09-21T17:34:58.466 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:58 smithi018 ceph-mon[63924]: Upgrade: Updating mds.foofs.smithi157.iijaxo 2024-09-21T17:34:58.466 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:58 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:58.466 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:58 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi157.iijaxo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T17:34:58.466 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:58 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:58.466 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:58 smithi018 ceph-mon[63924]: Deploying daemon mds.foofs.smithi157.iijaxo on smithi157 2024-09-21T17:34:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:58 smithi157 ceph-mon[49460]: Upgrade: Updating mds.foofs.smithi157.iijaxo 2024-09-21T17:34:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:58 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:34:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:58 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi157.iijaxo", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T17:34:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:58 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:34:58.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:58 smithi157 ceph-mon[49460]: Deploying daemon mds.foofs.smithi157.iijaxo on smithi157 2024-09-21T17:34:58.840 INFO:teuthology.orchestra.run.smithi018.stdout: 5 265 23.16 MB/sec execute 1 sec latency 334.043 ms 2024-09-21T17:34:59.250 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:59 smithi018 ceph-mon[63924]: pgmap v303: 97 pgs: 97 active+clean; 355 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 431 op/s 2024-09-21T17:34:59.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:34:59 smithi157 ceph-mon[49460]: pgmap v303: 97 pgs: 97 active+clean; 355 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 431 op/s 2024-09-21T17:34:59.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:34:59 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[63920]: 2024-09-21T17:34:59.324+0000 7f91b084c640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:34:59.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:34:59 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:34:59.347+0000 7f35fc230640 -1 bdev(0x55c98043cc00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 3 2024-09-21T17:34:59.840 INFO:teuthology.orchestra.run.smithi018.stdout: 5 342 17.43 MB/sec execute 2 sec latency 528.251 ms 2024-09-21T17:35:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:00 smithi157 ceph-mon[49460]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T17:35:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:00 smithi157 ceph-mon[49460]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:35:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:00 smithi157 ceph-mon[49460]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T17:35:00.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:00 smithi157 ceph-mon[49460]: Standby daemon mds.foofs.smithi018.wzitfj assigned to filesystem foofs as rank 0 2024-09-21T17:35:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:00 smithi157 ceph-mon[49460]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T17:35:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:00 smithi157 ceph-mon[49460]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T17:35:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:00 smithi157 ceph-mon[49460]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T17:35:00.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:00 smithi157 ceph-mon[49460]: fsmap foofs:1/1 {0=foofs.smithi018.wzitfj=up:replay} 2024-09-21T17:35:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:00 smithi018 ceph-mon[63924]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T17:35:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:00 smithi018 ceph-mon[63924]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T17:35:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:00 smithi018 ceph-mon[63924]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T17:35:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:00 smithi018 ceph-mon[63924]: Standby daemon mds.foofs.smithi018.wzitfj assigned to filesystem foofs as rank 0 2024-09-21T17:35:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:00 smithi018 ceph-mon[63924]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T17:35:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:00 smithi018 ceph-mon[63924]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T17:35:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:00 smithi018 ceph-mon[63924]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T17:35:00.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:00 smithi018 ceph-mon[63924]: fsmap foofs:1/1 {0=foofs.smithi018.wzitfj=up:replay} 2024-09-21T17:35:00.840 INFO:teuthology.orchestra.run.smithi018.stdout: 5 342 11.62 MB/sec execute 3 sec latency 1528.304 ms 2024-09-21T17:35:01.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:01 smithi018 ceph-mon[63924]: pgmap v305: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 406 op/s 2024-09-21T17:35:01.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:01.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:35:01.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"} : dispatch 2024-09-21T17:35:01.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:01 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:reconnect 2024-09-21T17:35:01.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:01 smithi018 ceph-mon[63924]: fsmap foofs:1/1 {0=foofs.smithi018.wzitfj=up:reconnect} 2024-09-21T17:35:01.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:01 smithi157 ceph-mon[49460]: pgmap v305: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 406 op/s 2024-09-21T17:35:01.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:01.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:35:01.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"} : dispatch 2024-09-21T17:35:01.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:01 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:reconnect 2024-09-21T17:35:01.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:01 smithi157 ceph-mon[49460]: fsmap foofs:1/1 {0=foofs.smithi018.wzitfj=up:reconnect} 2024-09-21T17:35:01.840 INFO:teuthology.orchestra.run.smithi018.stdout: 5 342 8.71 MB/sec execute 4 sec latency 2528.374 ms 2024-09-21T17:35:02.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:02 smithi018 ceph-mon[63924]: reconnect by client.14554 172.21.15.18:0/358269258 after 0 2024-09-21T17:35:02.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:02 smithi018 ceph-mon[63924]: reconnect by client.24347 172.21.15.157:0/265609017 after 0 2024-09-21T17:35:02.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:02 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:rejoin 2024-09-21T17:35:02.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:02 smithi018 ceph-mon[63924]: fsmap foofs:1/1 {0=foofs.smithi018.wzitfj=up:rejoin} 2024-09-21T17:35:02.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:02 smithi157 ceph-mon[49460]: reconnect by client.14554 172.21.15.18:0/358269258 after 0 2024-09-21T17:35:02.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:02 smithi157 ceph-mon[49460]: reconnect by client.24347 172.21.15.157:0/265609017 after 0 2024-09-21T17:35:02.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:02 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:rejoin 2024-09-21T17:35:02.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:02 smithi157 ceph-mon[49460]: fsmap foofs:1/1 {0=foofs.smithi018.wzitfj=up:rejoin} 2024-09-21T17:35:02.840 INFO:teuthology.orchestra.run.smithi018.stdout: 5 cleanup 5 sec 2024-09-21T17:35:03.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:03 smithi018 ceph-mon[63924]: pgmap v306: 97 pgs: 97 active+clean; 300 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 8.8 MiB/s wr, 426 op/s 2024-09-21T17:35:03.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:03 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:clientreplay 2024-09-21T17:35:03.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:03 smithi018 ceph-mon[63924]: fsmap foofs:1/1 {0=foofs.smithi018.wzitfj=up:clientreplay} 2024-09-21T17:35:03.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:03 smithi018 ceph-mon[63924]: daemon mds.foofs.smithi018.wzitfj is now active in filesystem foofs as rank 0 2024-09-21T17:35:03.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:03 smithi157 ceph-mon[49460]: pgmap v306: 97 pgs: 97 active+clean; 300 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 8.8 MiB/s wr, 426 op/s 2024-09-21T17:35:03.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:03 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:clientreplay 2024-09-21T17:35:03.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:03 smithi157 ceph-mon[49460]: fsmap foofs:1/1 {0=foofs.smithi018.wzitfj=up:clientreplay} 2024-09-21T17:35:03.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:03 smithi157 ceph-mon[49460]: daemon mds.foofs.smithi018.wzitfj is now active in filesystem foofs as rank 0 2024-09-21T17:35:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: 5 cleanup 6 sec 2024-09-21T17:35:03.982 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:35:04.449 INFO:teuthology.orchestra.run.smithi018.stdout: 0 cleanup 7 sec 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout: Operation Count AvgLat MaxLat 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout: ---------------------------------------- 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout: NTCreateX 158 109.409 4938.745 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout: Close 159 45.093 4931.908 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout: Qfileinfo 132 57.551 4893.125 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout: WriteX 663 0.031 0.084 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:35:04.450 INFO:teuthology.orchestra.run.smithi018.stdout:Throughput 8.71486 MB/sec 5 clients 5 procs max_latency=2528.374 ms 2024-09-21T17:35:04.452 DEBUG:teuthology.parallel:result is None 2024-09-21T17:35:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:04 smithi018 ceph-mon[63924]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T17:35:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:04 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:active 2024-09-21T17:35:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:04 smithi018 ceph-mon[63924]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 2024-09-21T17:35:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:04 smithi157 ceph-mon[49460]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T17:35:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:04 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809] up:active 2024-09-21T17:35:04.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:04 smithi157 ceph-mon[49460]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (9m) 16s ago 18m 21.0M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (6m) 16s ago 18m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (6m) 44s ago 18m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (9m) 16s ago 18m 48.3M - dad864ee21e9 48d78bd31599 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (24s) 16s ago 12m 10.9M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 33ae5dc72cb9 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (12m) 44s ago 12m 16.8M - 17.2.0 e1d6a67b021e 929ad8558ce8 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (8m) 16s ago 20m 523M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (8m) 44s ago 18m 459M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:35:04.804 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (7m) 16s ago 20m 60.0M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (7m) 44s ago 18m 45.9M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (12m) 16s ago 12m 283M - 4.0 e1d6a67b021e 554eeb48d706 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (12m) 44s ago 12m 87.4M - 4.0 e1d6a67b021e df4b64c06e72 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (9m) 16s ago 18m 10.2M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (8m) 44s ago 17m 11.7M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (5m) 16s ago 17m 371M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (5m) 16s ago 16m 325M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (4m) 16s ago 16m 274M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (3m) 16s ago 15m 364M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (2m) 44s ago 15m 333M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 2552dce6837e 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (2m) 44s ago 15m 272M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d abad654e4600 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (86s) 44s ago 14m 238M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d fd2e6eb21352 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (49s) 44s ago 14m 16.5M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 85bf07d224b1 2024-09-21T17:35:04.805 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (8m) 16s ago 17m 57.8M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:35:05.240 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:35:05.240 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:35:05.240 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:35:05.240 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:35:05.240 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:35:05.240 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:35:05.240 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:35:05.240 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 8 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 1 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 13 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:35:05.241 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "osd", 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "crash", 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "mon" 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "15/23 daemons upgraded", 2024-09-21T17:35:05.629 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T17:35:05.630 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:35:05.630 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:35:05.630 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:05 smithi018 ceph-mon[63924]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:05.630 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:05 smithi018 ceph-mon[63924]: pgmap v307: 97 pgs: 97 active+clean; 298 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.5 MiB/s wr, 302 op/s 2024-09-21T17:35:05.630 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:05 smithi018 ceph-mon[63924]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:05.630 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:05 smithi018 ceph-mon[63924]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:05.630 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:05 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/513564758' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:05.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:05 smithi157 ceph-mon[49460]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:05.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:05 smithi157 ceph-mon[49460]: pgmap v307: 97 pgs: 97 active+clean; 298 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.5 MiB/s wr, 302 op/s 2024-09-21T17:35:05.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:05 smithi157 ceph-mon[49460]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:05.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:05 smithi157 ceph-mon[49460]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:05.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:05 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/513564758' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:06.094 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_WARN insufficient standby MDS daemons available 2024-09-21T17:35:06.095 INFO:teuthology.orchestra.run.smithi018.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-21T17:35:06.095 INFO:teuthology.orchestra.run.smithi018.stdout: have 0; want 1 more 2024-09-21T17:35:06.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:06 smithi018 ceph-mon[63924]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:06.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:06 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/45424291' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:35:06.646 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:06 smithi018 ceph-mon[63924]: pgmap v308: 97 pgs: 97 active+clean; 298 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.1 MiB/s wr, 281 op/s 2024-09-21T17:35:06.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:06 smithi157 ceph-mon[49460]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:06.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:06 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/45424291' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:35:06.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:06 smithi157 ceph-mon[49460]: pgmap v308: 97 pgs: 97 active+clean; 298 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.1 MiB/s wr, 281 op/s 2024-09-21T17:35:09.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:09 smithi157 ceph-mon[49460]: pgmap v309: 97 pgs: 97 active+clean; 280 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 6.5 MiB/s wr, 176 op/s 2024-09-21T17:35:09.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:09 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:09.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:09 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:09.279 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:09 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:09.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:09 smithi018 ceph-mon[63924]: pgmap v309: 97 pgs: 97 active+clean; 280 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 6.5 MiB/s wr, 176 op/s 2024-09-21T17:35:09.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:09 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:09.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:09 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:09.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:09 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:10.486 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:10 smithi157 ceph-mon[49460]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T17:35:10.486 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:10 smithi157 ceph-mon[49460]: Cluster is now healthy 2024-09-21T17:35:10.486 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:10 smithi157 ceph-mon[49460]: mds.? [v2:172.21.15.157:6832/3217782690,v1:172.21.15.157:6833/3217782690] up:boot 2024-09-21T17:35:10.486 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:10 smithi157 ceph-mon[49460]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:35:10.487 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:10 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"} : dispatch 2024-09-21T17:35:10.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:10 smithi018 ceph-mon[63924]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T17:35:10.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:10 smithi018 ceph-mon[63924]: Cluster is now healthy 2024-09-21T17:35:10.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:10 smithi018 ceph-mon[63924]: mds.? [v2:172.21.15.157:6832/3217782690,v1:172.21.15.157:6833/3217782690] up:boot 2024-09-21T17:35:10.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:10 smithi018 ceph-mon[63924]: fsmap foofs:1 {0=foofs.smithi018.wzitfj=up:active} 1 up:standby 2024-09-21T17:35:10.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:10 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "mds metadata", "who": "foofs.smithi157.iijaxo"} : dispatch 2024-09-21T17:35:11.460 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:11 smithi157 ceph-mon[49460]: pgmap v310: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.8 MiB/s wr, 129 op/s 2024-09-21T17:35:11.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:11 smithi018 ceph-mon[63924]: pgmap v310: 97 pgs: 97 active+clean; 265 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.8 MiB/s wr, 129 op/s 2024-09-21T17:35:13.094 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:12 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:13.094 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:12 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:13.094 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:12 smithi157 ceph-mon[49460]: pgmap v311: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.3 MiB/s wr, 123 op/s 2024-09-21T17:35:13.094 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:12 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:13.094 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:12 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:13.152 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:12 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:13.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:12 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:13.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:12 smithi018 ceph-mon[63924]: pgmap v311: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 4.3 MiB/s wr, 123 op/s 2024-09-21T17:35:13.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:12 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:13.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:12 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:15 smithi157 ceph-mon[49460]: pgmap v312: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 752 KiB/s wr, 75 op/s 2024-09-21T17:35:15.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:15 smithi018 ceph-mon[63924]: pgmap v312: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 752 KiB/s wr, 75 op/s 2024-09-21T17:35:16.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.903 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: Detected new or changed devices on smithi157 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: pgmap v313: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 107 KiB/s wr, 35 op/s 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.904 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all mds 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi018.wzitfj"} : dispatch 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi018.wzitfj"}]': finished 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi157.iijaxo"} : dispatch 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi157.iijaxo"}]': finished 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all rgw 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.905 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T17:35:16.906 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:16.906 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:16.906 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all iscsi 2024-09-21T17:35:16.906 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: Detected new or changed devices on smithi157 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: pgmap v313: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 107 KiB/s wr, 35 op/s 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.029 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all mds 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi018.wzitfj"} : dispatch 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi018.wzitfj"}]': finished 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi157.iijaxo"} : dispatch 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi157.iijaxo"}]': finished 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all rgw 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.030 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T17:35:17.031 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.031 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.031 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.031 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T17:35:17.031 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:17.031 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:17.031 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all iscsi 2024-09-21T17:35:17.031 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:19.461 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: pgmap v314: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 106 KiB/s wr, 32 op/s 2024-09-21T17:35:19.461 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:19.461 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T17:35:19.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T17:35:19.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T17:35:19.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:19.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T17:35:19.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T17:35:19.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T17:35:19.462 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:19 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:19.519 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: pgmap v314: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 106 KiB/s wr, 32 op/s 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi018.rwskmy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T17:35:19.520 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:19 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:20.232 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:20 smithi018 ceph-mon[63924]: Upgrade: Updating nfs.foo.0.0.smithi018.rwskmy (1/2) 2024-09-21T17:35:20.232 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:20 smithi018 ceph-mon[63924]: Creating key for client.nfs.foo.0.0.smithi018.rwskmy 2024-09-21T17:35:20.232 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:20 smithi018 ceph-mon[63924]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T17:35:20.232 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:20 smithi018 ceph-mon[63924]: Rados config object exists: conf-nfs.foo 2024-09-21T17:35:20.232 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:20 smithi018 ceph-mon[63924]: Creating key for client.nfs.foo.0.0.smithi018.rwskmy-rgw 2024-09-21T17:35:20.232 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:20 smithi018 ceph-mon[63924]: Bind address in nfs.foo.0.0.smithi018.rwskmy's ganesha conf is defaulting to empty 2024-09-21T17:35:20.232 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:20 smithi018 ceph-mon[63924]: Deploying daemon nfs.foo.0.0.smithi018.rwskmy on smithi018 2024-09-21T17:35:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:20 smithi157 ceph-mon[49460]: Upgrade: Updating nfs.foo.0.0.smithi018.rwskmy (1/2) 2024-09-21T17:35:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:20 smithi157 ceph-mon[49460]: Creating key for client.nfs.foo.0.0.smithi018.rwskmy 2024-09-21T17:35:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:20 smithi157 ceph-mon[49460]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T17:35:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:20 smithi157 ceph-mon[49460]: Rados config object exists: conf-nfs.foo 2024-09-21T17:35:20.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:20 smithi157 ceph-mon[49460]: Creating key for client.nfs.foo.0.0.smithi018.rwskmy-rgw 2024-09-21T17:35:20.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:20 smithi157 ceph-mon[49460]: Bind address in nfs.foo.0.0.smithi018.rwskmy's ganesha conf is defaulting to empty 2024-09-21T17:35:20.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:20 smithi157 ceph-mon[49460]: Deploying daemon nfs.foo.0.0.smithi018.rwskmy on smithi018 2024-09-21T17:35:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:21 smithi157 ceph-mon[49460]: pgmap v315: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 104 KiB/s wr, 24 op/s 2024-09-21T17:35:21.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:21 smithi018 ceph-mon[63924]: pgmap v315: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 104 KiB/s wr, 24 op/s 2024-09-21T17:35:23.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:23 smithi157 ceph-mon[49460]: pgmap v316: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 1.6 KiB/s wr, 7 op/s 2024-09-21T17:35:23.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:23 smithi157 ceph-mon[49460]: from='mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809]' entity='mds.foofs.smithi018.wzitfj' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.18:0/358269258"} : dispatch 2024-09-21T17:35:23.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:23 smithi018 ceph-mon[63924]: pgmap v316: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 1.6 KiB/s wr, 7 op/s 2024-09-21T17:35:23.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:23 smithi018 ceph-mon[63924]: from='mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809]' entity='mds.foofs.smithi018.wzitfj' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.18:0/358269258"} : dispatch 2024-09-21T17:35:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:24 smithi157 ceph-mon[49460]: Evicting (and blocklisting) client session 14554 (172.21.15.18:0/358269258) 2024-09-21T17:35:24.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:24 smithi157 ceph-mon[49460]: from='mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809]' entity='mds.foofs.smithi018.wzitfj' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.18:0/358269258"}]': finished 2024-09-21T17:35:24.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:24 smithi157 ceph-mon[49460]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T17:35:24.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:24.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:24 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:24.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:24 smithi018 ceph-mon[63924]: Evicting (and blocklisting) client session 14554 (172.21.15.18:0/358269258) 2024-09-21T17:35:24.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:24 smithi018 ceph-mon[63924]: from='mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809]' entity='mds.foofs.smithi018.wzitfj' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.18:0/358269258"}]': finished 2024-09-21T17:35:24.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:24 smithi018 ceph-mon[63924]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T17:35:24.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:24.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:24 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: pgmap v318: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T17:35:25.323 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:25 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: pgmap v318: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T17:35:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T17:35:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T17:35:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T17:35:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:25.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T17:35:25.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T17:35:25.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi157.pgvtbt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T17:35:25.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:25 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:26.434 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:26 smithi157 ceph-mon[49460]: Upgrade: Updating nfs.foo.1.0.smithi157.pgvtbt (2/2) 2024-09-21T17:35:26.434 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:26 smithi157 ceph-mon[49460]: Creating key for client.nfs.foo.1.0.smithi157.pgvtbt 2024-09-21T17:35:26.434 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:26 smithi157 ceph-mon[49460]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T17:35:26.434 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:26 smithi157 ceph-mon[49460]: Rados config object exists: conf-nfs.foo 2024-09-21T17:35:26.434 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:26 smithi157 ceph-mon[49460]: Creating key for client.nfs.foo.1.0.smithi157.pgvtbt-rgw 2024-09-21T17:35:26.434 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:26 smithi157 ceph-mon[49460]: Bind address in nfs.foo.1.0.smithi157.pgvtbt's ganesha conf is defaulting to empty 2024-09-21T17:35:26.434 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:26 smithi157 ceph-mon[49460]: Deploying daemon nfs.foo.1.0.smithi157.pgvtbt on smithi157 2024-09-21T17:35:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:26 smithi018 ceph-mon[63924]: Upgrade: Updating nfs.foo.1.0.smithi157.pgvtbt (2/2) 2024-09-21T17:35:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:26 smithi018 ceph-mon[63924]: Creating key for client.nfs.foo.1.0.smithi157.pgvtbt 2024-09-21T17:35:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:26 smithi018 ceph-mon[63924]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T17:35:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:26 smithi018 ceph-mon[63924]: Rados config object exists: conf-nfs.foo 2024-09-21T17:35:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:26 smithi018 ceph-mon[63924]: Creating key for client.nfs.foo.1.0.smithi157.pgvtbt-rgw 2024-09-21T17:35:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:26 smithi018 ceph-mon[63924]: Bind address in nfs.foo.1.0.smithi157.pgvtbt's ganesha conf is defaulting to empty 2024-09-21T17:35:26.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:26 smithi018 ceph-mon[63924]: Deploying daemon nfs.foo.1.0.smithi157.pgvtbt on smithi157 2024-09-21T17:35:27.507 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:27 smithi157 ceph-mon[49460]: pgmap v319: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T17:35:27.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:27 smithi018 ceph-mon[63924]: pgmap v319: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T17:35:29.427 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:29 smithi157 ceph-mon[49460]: pgmap v320: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T17:35:29.427 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:29 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:29.427 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:29 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:29.427 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:29 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:29.503 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:29 smithi018 ceph-mon[63924]: pgmap v320: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T17:35:29.503 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:29 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:29.503 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:29 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:29.503 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:29 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:31.520 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:31 smithi018 ceph-mon[63924]: pgmap v321: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:31.520 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:35:31.523 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:31 smithi157 ceph-mon[49460]: pgmap v321: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:31.523 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:35:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:32 smithi157 ceph-mon[49460]: Evicting (and blocklisting) client session 24347 (172.21.15.157:0/265609017) 2024-09-21T17:35:32.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:32 smithi157 ceph-mon[49460]: from='mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809]' entity='mds.foofs.smithi018.wzitfj' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.157:0/265609017"} : dispatch 2024-09-21T17:35:32.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:32 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:32.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:32 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:32.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:32 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:32.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:32 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:32.583 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:32 smithi018 ceph-mon[63924]: Evicting (and blocklisting) client session 24347 (172.21.15.157:0/265609017) 2024-09-21T17:35:32.584 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:32 smithi018 ceph-mon[63924]: from='mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809]' entity='mds.foofs.smithi018.wzitfj' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.157:0/265609017"} : dispatch 2024-09-21T17:35:32.584 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:32 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:32.584 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:32 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:32.584 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:32 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:32.584 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:32 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:33 smithi157 ceph-mon[49460]: pgmap v322: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:33 smithi157 ceph-mon[49460]: from='mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809]' entity='mds.foofs.smithi018.wzitfj' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.157:0/265609017"}]': finished 2024-09-21T17:35:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:33 smithi157 ceph-mon[49460]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T17:35:33.559 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:33 smithi018 ceph-mon[63924]: pgmap v322: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:33.559 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:33 smithi018 ceph-mon[63924]: from='mds.? [v2:172.21.15.18:6834/3948320809,v1:172.21.15.18:6835/3948320809]' entity='mds.foofs.smithi018.wzitfj' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.157:0/265609017"}]': finished 2024-09-21T17:35:33.559 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:33 smithi018 ceph-mon[63924]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T17:35:35.029 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:34 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:35.030 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:34 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:35.030 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:34 smithi018 ceph-mon[63924]: pgmap v324: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:35.030 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:34 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:35.030 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:34 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:35.270 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:34 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:35.271 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:34 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:35.271 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:34 smithi157 ceph-mon[49460]: pgmap v324: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:35.271 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:34 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:35.271 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:34 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:36.533 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:35:36.564 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:36 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:36.564 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:36 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:36.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:36 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:36.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:36 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:37.274 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:35:37.274 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (10m) 3s ago 19m 21.0M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:35:37.274 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (7m) 3s ago 19m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:35:37.274 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (6m) 5s ago 18m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:35:37.274 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (10m) 3s ago 19m 55.2M - dad864ee21e9 48d78bd31599 2024-09-21T17:35:37.274 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (57s) 3s ago 13m 38.4M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 33ae5dc72cb9 2024-09-21T17:35:37.274 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (29s) 5s ago 13m 14.7M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 0d5a7f74b1ff 2024-09-21T17:35:37.274 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (9m) 3s ago 20m 525M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (8m) 5s ago 18m 459M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (8m) 3s ago 20m 63.1M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (7m) 5s ago 18m 48.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (14s) 3s ago 12m 64.5M - 5.9 3fa9090ef91d 968c03310f33 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (9s) 5s ago 12m 17.7M - 5.9 3fa9090ef91d 759a07979654 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (9m) 3s ago 19m 11.2M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (9m) 5s ago 18m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (6m) 3s ago 17m 399M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (5m) 3s ago 17m 342M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (4m) 3s ago 16m 298M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (4m) 3s ago 16m 389M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (3m) 5s ago 16m 366M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 2552dce6837e 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (2m) 5s ago 15m 307M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d abad654e4600 2024-09-21T17:35:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (119s) 5s ago 15m 280M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d fd2e6eb21352 2024-09-21T17:35:37.276 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (81s) 5s ago 14m 309M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 85bf07d224b1 2024-09-21T17:35:37.276 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (8m) 3s ago 18m 57.8M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:35:37.525 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:37 smithi018 ceph-mon[63924]: pgmap v325: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:37.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:37 smithi157 ceph-mon[49460]: pgmap v325: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:37.779 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:35:37.779 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:35:37.779 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:35:37.779 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:35:37.779 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:35:37.779 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 8 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 14 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:35:37.780 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:35:38.153 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:35:38.153 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", 2024-09-21T17:35:38.153 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": true, 2024-09-21T17:35:38.153 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T17:35:38.153 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [ 2024-09-21T17:35:38.153 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr", 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: "mds", 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: "mon", 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: "nfs", 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: "osd", 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: "crash" 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": "18/23 daemons upgraded", 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "Currently upgrading nfs daemons", 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:35:38.154 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:35:38.612 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:35:38.612 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:38 smithi018 ceph-mon[63924]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:38.612 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:38 smithi018 ceph-mon[63924]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:38.612 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:38 smithi018 ceph-mon[63924]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:38.612 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:38 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/728947999' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:38.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:38 smithi157 ceph-mon[49460]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:38.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:38 smithi157 ceph-mon[49460]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:38.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:38 smithi157 ceph-mon[49460]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:38.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:38 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/728947999' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:39.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:39.319 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3695588102' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.320 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi018.rwskmy"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi018.rwskmy"}]': finished 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi157.pgvtbt"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi157.pgvtbt"}]': finished 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.321 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.322 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:39 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.779 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3695588102' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.780 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi018.rwskmy"} : dispatch 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi018.rwskmy"}]': finished 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi157.pgvtbt"} : dispatch 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi157.pgvtbt"}]': finished 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:39.781 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:39 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:35:40.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:40 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all nfs 2024-09-21T17:35:40.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:40 smithi018 ceph-mon[63924]: Upgrade: Setting container_image for all nvmeof 2024-09-21T17:35:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:40 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all nfs 2024-09-21T17:35:40.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:40 smithi157 ceph-mon[49460]: Upgrade: Setting container_image for all nvmeof 2024-09-21T17:35:41.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:41 smithi018 ceph-mon[63924]: pgmap v327: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T17:35:41.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:41 smithi157 ceph-mon[49460]: pgmap v327: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T17:35:42.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:42 smithi018 ceph-mon[63924]: Upgrade: Updating grafana.smithi018 2024-09-21T17:35:42.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:42 smithi018 ceph-mon[63924]: Deploying daemon grafana.smithi018 on smithi018 2024-09-21T17:35:42.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:42 smithi157 ceph-mon[49460]: Upgrade: Updating grafana.smithi018 2024-09-21T17:35:42.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:42 smithi157 ceph-mon[49460]: Deploying daemon grafana.smithi018 on smithi018 2024-09-21T17:35:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:43 smithi018 ceph-mon[63924]: pgmap v328: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-21T17:35:43.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:43 smithi157 ceph-mon[49460]: pgmap v328: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-21T17:35:45.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:45 smithi018 ceph-mon[63924]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-09-21T17:35:45.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:45 smithi157 ceph-mon[49460]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-09-21T17:35:46.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:35:46.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:35:47.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:47 smithi018 ceph-mon[63924]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:47.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:47 smithi157 ceph-mon[49460]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:49.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:49 smithi018 ceph-mon[63924]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:49.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:49 smithi157 ceph-mon[49460]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:51.636 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:51 smithi018 ceph-mon[63924]: pgmap v332: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:51.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:51 smithi157 ceph-mon[49460]: pgmap v332: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:53.491 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:53 smithi018 ceph-mon[63924]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:53.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:53 smithi157 ceph-mon[49460]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:54.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:54.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:54.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:54 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:54.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:54 smithi018 ceph-mon[63924]: pgmap v334: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:35:54.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:54.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:54.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:54 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:35:54.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:54 smithi157 ceph-mon[49460]: pgmap v334: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:35:57.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:57 smithi018 ceph-mon[63924]: pgmap v335: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:57.481 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:57 smithi157 ceph-mon[49460]: pgmap v335: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:59 smithi157 ceph-mon[49460]: pgmap v336: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:59 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:59 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:59 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:59.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:35:59 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:59.531 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:59 smithi018 ceph-mon[63924]: pgmap v336: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:35:59.531 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:59 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:59.531 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:59 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:59.531 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:59 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:35:59.531 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:35:59 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:01.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:01 smithi018 ceph-mon[63924]: pgmap v337: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:01.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:36:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:01 smithi157 ceph-mon[49460]: pgmap v337: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: pgmap v338: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T17:36:03.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:03 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T17:36:03.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: pgmap v338: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:03.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T17:36:03.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T17:36:03.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:03.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T17:36:03.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T17:36:03.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T17:36:03.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:03 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: Upgrade: Finalizing container_image settings 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.530 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: Upgrade: Complete! 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:04.531 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:04 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T17:36:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: Upgrade: Finalizing container_image settings 2024-09-21T17:36:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T17:36:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T17:36:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T17:36:04.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: Upgrade: Complete! 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:36:04.655 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:04.656 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:04 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:05 smithi157 ceph-mon[49460]: pgmap v339: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:05 smithi018 ceph-mon[63924]: pgmap v339: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:06 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:07.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:06 smithi157 ceph-mon[49460]: pgmap v340: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:07.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:06 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:36:07.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:06 smithi018 ceph-mon[63924]: pgmap v340: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:09.300 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:09 smithi018 ceph-mon[63924]: pgmap v341: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:09 smithi157 ceph-mon[49460]: pgmap v341: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:09.712 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph orch ps' 2024-09-21T17:36:10.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:10 smithi018 ceph-mon[63924]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:36:10.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:10 smithi157 ceph-mon[49460]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:36:11.507 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:11 smithi018 ceph-mon[63924]: pgmap v342: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:11 smithi157 ceph-mon[49460]: pgmap v342: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:12.252 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (10m) 13s ago 20m 21.0M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (7m) 13s ago 19m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (7m) 40s ago 19m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (19s) 13s ago 19m 36.8M - 10.4.8 5aad1d7cf004 3eca4a52b9bd 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (92s) 13s ago 13m 38.4M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 33ae5dc72cb9 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (64s) 40s ago 13m 14.7M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 0d5a7f74b1ff 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (9m) 13s ago 21m 533M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (9m) 40s ago 19m 459M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (8m) 13s ago 21m 63.8M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (8m) 40s ago 19m 48.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (49s) 13s ago 13m 65.2M - 5.9 3fa9090ef91d 968c03310f33 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (44s) 40s ago 13m 17.7M - 5.9 3fa9090ef91d 759a07979654 2024-09-21T17:36:12.253 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (10m) 13s ago 19m 12.2M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (10m) 40s ago 19m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (6m) 13s ago 18m 400M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (6m) 13s ago 18m 342M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (5m) 13s ago 17m 299M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (4m) 13s ago 17m 390M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (3m) 40s ago 16m 366M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 2552dce6837e 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (3m) 40s ago 16m 307M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d abad654e4600 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (2m) 40s ago 15m 280M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d fd2e6eb21352 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (116s) 40s ago 15m 309M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 85bf07d224b1 2024-09-21T17:36:12.254 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (9m) 13s ago 18m 57.8M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:36:12.905 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph versions' 2024-09-21T17:36:13.494 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:13 smithi018 ceph-mon[63924]: pgmap v343: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:13.494 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:13 smithi018 ceph-mon[63924]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:36:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:13 smithi157 ceph-mon[49460]: pgmap v343: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:13 smithi157 ceph-mon[49460]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:36:15.032 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 8 2024-09-21T17:36:15.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:36:15.033 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:36:15.033 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:36:15.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:36:15.033 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:36:15.033 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 14 2024-09-21T17:36:15.033 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:36:15.033 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:36:15.371 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:15 smithi018 ceph-mon[63924]: pgmap v344: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:15.371 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:15 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2438178883' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:15 smithi157 ceph-mon[49460]: pgmap v344: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:15 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2438178883' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:36:15.715 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-21T17:36:16.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:36:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:36:17.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:17 smithi018 ceph-mon[63924]: pgmap v345: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:17 smithi157 ceph-mon[49460]: pgmap v345: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:19.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:19 smithi018 ceph-mon[63924]: pgmap v346: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:19 smithi157 ceph-mon[49460]: pgmap v346: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:20.685 INFO:teuthology.orchestra.run.smithi018.stdout:wait for servicemap items w/ changing names to refresh 2024-09-21T17:36:21.442 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:21 smithi018 ceph-mon[63924]: pgmap v347: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:21.462 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'sleep 60' 2024-09-21T17:36:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:21 smithi157 ceph-mon[49460]: pgmap v347: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:23.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:23 smithi018 ceph-mon[63924]: pgmap v348: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:23 smithi157 ceph-mon[49460]: pgmap v348: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:25 smithi157 ceph-mon[49460]: pgmap v349: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:25.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:25 smithi018 ceph-mon[63924]: pgmap v349: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:27 smithi157 ceph-mon[49460]: pgmap v350: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:27.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:27 smithi018 ceph-mon[63924]: pgmap v350: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:29.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:29 smithi157 ceph-mon[49460]: pgmap v351: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:29.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:29 smithi018 ceph-mon[63924]: pgmap v351: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:31 smithi157 ceph-mon[49460]: pgmap v352: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:36:31.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:31 smithi018 ceph-mon[63924]: pgmap v352: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:31.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:36:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:33 smithi157 ceph-mon[49460]: pgmap v353: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:33.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:33 smithi018 ceph-mon[63924]: pgmap v353: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:35 smithi157 ceph-mon[49460]: pgmap v354: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:35.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:35 smithi018 ceph-mon[63924]: pgmap v354: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:37.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:37 smithi157 ceph-mon[49460]: pgmap v355: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:37.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:37 smithi018 ceph-mon[63924]: pgmap v355: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:39 smithi157 ceph-mon[49460]: pgmap v356: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:39.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:39 smithi018 ceph-mon[63924]: pgmap v356: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:41.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:41 smithi157 ceph-mon[49460]: pgmap v357: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:41.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:41 smithi018 ceph-mon[63924]: pgmap v357: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:43 smithi157 ceph-mon[49460]: pgmap v358: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:43.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:43 smithi018 ceph-mon[63924]: pgmap v358: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:45 smithi157 ceph-mon[49460]: pgmap v359: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:45.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:45 smithi018 ceph-mon[63924]: pgmap v359: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:46.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:36:46.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:36:47.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:47 smithi157 ceph-mon[49460]: pgmap v360: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:47.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:47 smithi018 ceph-mon[63924]: pgmap v360: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:49.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:49 smithi157 ceph-mon[49460]: pgmap v361: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:49.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:49 smithi018 ceph-mon[63924]: pgmap v361: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:51.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:51 smithi157 ceph-mon[49460]: pgmap v362: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:51 smithi018 ceph-mon[63924]: pgmap v362: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:53.028 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:52 smithi157 ceph-mon[49460]: pgmap v363: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:53.153 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:52 smithi018 ceph-mon[63924]: pgmap v363: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:55.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:55 smithi157 ceph-mon[49460]: pgmap v364: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:55.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:55 smithi018 ceph-mon[63924]: pgmap v364: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:36:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:57 smithi157 ceph-mon[49460]: pgmap v365: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:57.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:57 smithi018 ceph-mon[63924]: pgmap v365: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:59.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:36:59 smithi157 ceph-mon[49460]: pgmap v366: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:36:59.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:36:59 smithi018 ceph-mon[63924]: pgmap v366: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:01 smithi157 ceph-mon[49460]: pgmap v367: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:01.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:01 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:37:01.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:01 smithi018 ceph-mon[63924]: pgmap v367: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:01.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:01 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:37:03.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:03 smithi157 ceph-mon[49460]: pgmap v368: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:03.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:03 smithi018 ceph-mon[63924]: pgmap v368: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: pgmap v369: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:05.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:37:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:37:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:37:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.529 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:05 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: pgmap v369: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T17:37:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T17:37:05.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T17:37:05.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:05.654 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:05 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' 2024-09-21T17:37:07.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:07 smithi157 ceph-mon[49460]: pgmap v370: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:07.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:07 smithi018 ceph-mon[63924]: pgmap v370: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:09.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:09 smithi157 ceph-mon[49460]: pgmap v371: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:09.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:09 smithi018 ceph-mon[63924]: pgmap v371: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:11.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:11 smithi157 ceph-mon[49460]: pgmap v372: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:11.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:11 smithi018 ceph-mon[63924]: pgmap v372: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:13.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:13 smithi157 ceph-mon[49460]: pgmap v373: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:13.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:13 smithi018 ceph-mon[63924]: pgmap v373: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:15.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:15 smithi157 ceph-mon[49460]: pgmap v374: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:15.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:15 smithi018 ceph-mon[63924]: pgmap v374: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:16.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:16 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:37:16.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:16 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:37:17.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:17 smithi157 ceph-mon[49460]: pgmap v375: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:17.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:17 smithi018 ceph-mon[63924]: pgmap v375: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:19.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:19 smithi157 ceph-mon[49460]: pgmap v376: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:19.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:19 smithi018 ceph-mon[63924]: pgmap v376: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:21.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:21 smithi157 ceph-mon[49460]: pgmap v377: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:21.652 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:21 smithi018 ceph-mon[63924]: pgmap v377: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:23.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:23 smithi157 ceph-mon[49460]: pgmap v378: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:23.534 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:23 smithi018 ceph-mon[63924]: pgmap v378: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:23.957 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph orch ps' 2024-09-21T17:37:25.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:25 smithi157 ceph-mon[49460]: pgmap v379: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:25.570 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:25 smithi018 ceph-mon[63924]: pgmap v379: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.smithi018 smithi018 *:9093,9094 running (12m) 87s ago 21m 21.0M - 0.27.0 11f11916f8cd 7dcb52b1a8cc 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi018 smithi018 running (8m) 87s ago 21m 6887k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 5041fc62af53 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:crash.smithi157 smithi157 running (8m) 114s ago 20m 6895k - 19.3.0-5129-g0ecbb435 3fa9090ef91d 240b1098d78b 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.smithi018 smithi018 *:3000 running (93s) 87s ago 21m 36.8M - 10.4.8 5aad1d7cf004 3eca4a52b9bd 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi018.wzitfj smithi018 running (2m) 87s ago 15m 38.4M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 33ae5dc72cb9 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foofs.smithi157.iijaxo smithi157 running (2m) 114s ago 15m 14.7M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 0d5a7f74b1ff 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi018.jndyyn smithi018 *:8443,9283,8765 running (11m) 87s ago 22m 533M - 19.3.0-5129-g0ecbb435 3fa9090ef91d df6e0745cc38 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.smithi157.oowooq smithi157 *:8443,9283,8765 running (10m) 114s ago 20m 459M - 19.3.0-5129-g0ecbb435 3fa9090ef91d 2ad4b873f99a 2024-09-21T17:37:26.223 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi018 smithi018 running (10m) 87s ago 22m 63.8M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d d7cfe2f8a92d 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:mon.smithi157 smithi157 running (9m) 114s ago 20m 48.7M 2048M 19.3.0-5129-g0ecbb435 3fa9090ef91d 1c6557eec7d0 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.0.0.smithi018.rwskmy smithi018 *:2049 running (2m) 87s ago 14m 65.2M - 5.9 3fa9090ef91d 968c03310f33 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:nfs.foo.1.0.smithi157.pgvtbt smithi157 *:2049 running (118s) 114s ago 14m 17.7M - 5.9 3fa9090ef91d 759a07979654 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi018 smithi018 *:9100 running (11m) 87s ago 20m 12.2M - 1.7.0 72c9c2088986 3e5bb06dd5ea 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.smithi157 smithi157 *:9100 running (11m) 114s ago 20m 11.5M - 1.7.0 72c9c2088986 e14998e3f938 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (8m) 87s ago 19m 400M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d ac5dc1252922 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (7m) 87s ago 19m 342M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 6ff6ff175840 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (6m) 87s ago 18m 299M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d 3df4e3f0df26 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (5m) 87s ago 18m 390M 1937M 19.3.0-5129-g0ecbb435 3fa9090ef91d e7c60a62ac9e 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi157 running (5m) 114s ago 17m 366M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 2552dce6837e 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi157 running (4m) 114s ago 17m 307M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d abad654e4600 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi157 running (3m) 114s ago 17m 280M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d fd2e6eb21352 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi157 running (3m) 114s ago 16m 309M 2704M 19.3.0-5129-g0ecbb435 3fa9090ef91d 85bf07d224b1 2024-09-21T17:37:26.224 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.smithi018 smithi018 *:9095 running (10m) 87s ago 20m 57.8M - 2.51.0 1d3b7f56885b ce2f0fb61e19 2024-09-21T17:37:26.858 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph orch upgrade status' 2024-09-21T17:37:27.470 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:27 smithi018 ceph-mon[63924]: pgmap v380: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:27.470 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:27 smithi018 ceph-mon[63924]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:37:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:27 smithi157 ceph-mon[49460]: pgmap v380: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:27.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:27 smithi157 ceph-mon[49460]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:37:28.901 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:37:28.901 INFO:teuthology.orchestra.run.smithi018.stdout: "target_image": null, 2024-09-21T17:37:28.901 INFO:teuthology.orchestra.run.smithi018.stdout: "in_progress": false, 2024-09-21T17:37:28.901 INFO:teuthology.orchestra.run.smithi018.stdout: "which": "", 2024-09-21T17:37:28.901 INFO:teuthology.orchestra.run.smithi018.stdout: "services_complete": [], 2024-09-21T17:37:28.901 INFO:teuthology.orchestra.run.smithi018.stdout: "progress": null, 2024-09-21T17:37:28.901 INFO:teuthology.orchestra.run.smithi018.stdout: "message": "", 2024-09-21T17:37:28.902 INFO:teuthology.orchestra.run.smithi018.stdout: "is_paused": false 2024-09-21T17:37:28.902 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:37:29.456 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:29 smithi018 ceph-mon[63924]: pgmap v381: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:29.483 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph health detail' 2024-09-21T17:37:29.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:29 smithi157 ceph-mon[49460]: pgmap v381: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:30.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:30 smithi018 ceph-mon[63924]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:37:30.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:30 smithi157 ceph-mon[49460]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:37:31.284 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:31 smithi018 ceph-mon[63924]: pgmap v382: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:31.285 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:31 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:37:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:31 smithi157 ceph-mon[49460]: pgmap v382: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:31.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:31 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:37:31.629 INFO:teuthology.orchestra.run.smithi018.stdout:HEALTH_OK 2024-09-21T17:37:32.198 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph versions' 2024-09-21T17:37:32.505 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:32 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2437081022' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:37:32.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:32 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2437081022' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T17:37:33.509 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:33 smithi018 ceph-mon[63924]: pgmap v383: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:33.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:33 smithi157 ceph-mon[49460]: pgmap v383: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:34.322 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2024-09-21T17:37:34.322 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2024-09-21T17:37:34.322 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 8 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 2 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)": 14 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:37:34.323 INFO:teuthology.orchestra.run.smithi018.stdout:} 2024-09-21T17:37:34.905 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-21T17:37:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:35 smithi157 ceph-mon[49460]: pgmap v384: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:35.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:35 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1716133217' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:37:35.540 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:35 smithi018 ceph-mon[63924]: pgmap v384: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:35.540 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:35 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1716133217' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:37:37.162 INFO:teuthology.orchestra.run.smithi018.stdout:true 2024-09-21T17:37:37.510 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:37 smithi018 ceph-mon[63924]: pgmap v385: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:37.510 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:37 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/3550340974' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:37:37.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:37 smithi157 ceph-mon[49460]: pgmap v385: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:37.600 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:37 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/3550340974' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:37:37.824 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -e sha1=0ecbb435437226bc091a0c2d4549ba25bf724efe -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-21T17:37:39.467 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:39 smithi018 ceph-mon[63924]: pgmap v386: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:39.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:39 smithi157 ceph-mon[49460]: pgmap v386: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:40.061 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 19.3.0-5129-g0ecbb435 (0ecbb435437226bc091a0c2d4549ba25bf724efe) squid (dev)" 2024-09-21T17:37:40.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:40 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/2253403417' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:37:40.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:40 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/2253403417' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T17:37:40.864 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-21T17:37:40.865 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-21T17:37:40.865 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T17:37:41.437 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:41 smithi018 ceph-mon[63924]: pgmap v387: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:41.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:41 smithi157 ceph-mon[49460]: pgmap v387: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:42.897 INFO:teuthology.orchestra.run.smithi018.stdout: 2024-09-21T17:37:42.898 INFO:teuthology.orchestra.run.smithi018.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T17:15:31.093066Z", "last_refresh": "2024-09-21T17:35:58.533289Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T17:15:24.971672Z", "last_refresh": "2024-09-21T17:35:31.537988Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-21T17:15:27.974654Z", "last_refresh": "2024-09-21T17:35:58.533434Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T17:22:16.273406Z", "last_refresh": "2024-09-21T17:35:31.538600Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T17:15:22.920160Z", "last_refresh": "2024-09-21T17:35:31.538090Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi018:172.21.15.18=smithi018", "smithi157:172.21.15.157=smithi157"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T17:16:15.589289Z", "last_refresh": "2024-09-21T17:35:31.538168Z", "running": 2, "size": 2}}, {"events": ["2024-09-21T17:37:04.884858Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-21T17:22:38.124746Z", "last_refresh": "2024-09-21T17:35:31.538672Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T17:15:29.301277Z", "last_refresh": "2024-09-21T17:35:31.538241Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5", "last_refresh": "2024-09-21T17:35:31.538314Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T17:15:26.707189Z", "last_refresh": "2024-09-21T17:35:58.533579Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T17:37:43.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:43 smithi157 ceph-mon[49460]: pgmap v388: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:43.581 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:43 smithi018 ceph-mon[63924]: pgmap v388: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:43.609 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-21T17:37:43.610 DEBUG:teuthology.parallel:result is None 2024-09-21T17:37:43.610 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T17:37:43.620 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:37:43.620 DEBUG:teuthology.orchestra.run.smithi018:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-21T17:37:43.649 INFO:teuthology.orchestra.run.smithi018.stderr:+ umount /mnt/foo 2024-09-21T17:37:43.679 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T17:37:43.689 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:37:43.689 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-21T17:37:44.360 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:44 smithi018 ceph-mon[63924]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:37:44.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:44 smithi157 ceph-mon[49460]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T17:37:45.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:45 smithi157 ceph-mon[49460]: pgmap v389: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:45.563 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:45 smithi018 ceph-mon[63924]: pgmap v389: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T17:37:45.869 INFO:teuthology.orchestra.run.smithi018.stdout: "foo" 2024-09-21T17:37:46.497 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:46 smithi018 ceph-mon[63924]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:37:46.526 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-21T17:37:46.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:46 smithi157 ceph-mon[49460]: from='mgr.34102 172.21.15.18:0/3389545749' entity='mgr.smithi018.jndyyn' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T17:37:47.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:47 smithi018 ceph-mon[63924]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:37:47.403 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:47 smithi018 ceph-mon[63924]: pgmap v390: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:47.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:47 smithi157 ceph-mon[49460]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:37:47.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:47 smithi157 ceph-mon[49460]: pgmap v390: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout:[ 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: "access_type": "RW", 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: "clients": [], 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: "cluster_id": "foo", 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: "export_id": 1, 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: "fsal": { 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: "fs_name": "foofs", 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: "name": "CEPH", 2024-09-21T17:37:48.652 INFO:teuthology.orchestra.run.smithi018.stdout: "user_id": "nfs.foo.1" 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: "path": "/", 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: "protocols": [ 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: 4 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: "pseudo": "/fake", 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: "security_label": true, 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: "squash": "none", 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: "transports": [ 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: "TCP" 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout: } 2024-09-21T17:37:48.653 INFO:teuthology.orchestra.run.smithi018.stdout:] 2024-09-21T17:37:49.279 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'rados -p .nfs --all ls -' 2024-09-21T17:37:49.586 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:49 smithi018 ceph-mon[63924]: pgmap v391: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:49.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:49 smithi157 ceph-mon[49460]: pgmap v391: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T17:37:50.569 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:50 smithi018 ceph-mon[63924]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:37:50.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:50 smithi157 ceph-mon[49460]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T17:37:51.123 INFO:teuthology.orchestra.run.smithi018.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-21T17:37:51.123 INFO:teuthology.orchestra.run.smithi018.stdout:foo grace 2024-09-21T17:37:51.123 INFO:teuthology.orchestra.run.smithi018.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-21T17:37:51.123 INFO:teuthology.orchestra.run.smithi018.stdout:foo conf-nfs.foo 2024-09-21T17:37:51.123 INFO:teuthology.orchestra.run.smithi018.stdout:foo export-1 2024-09-21T17:37:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:51 smithi018 ceph-mon[63924]: pgmap v392: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-21T17:37:51.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:51 smithi018 ceph-mon[63924]: mgrmap e44: smithi018.jndyyn(active, since 10m), standbys: smithi157.oowooq 2024-09-21T17:37:51.724 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T17:37:51.733 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:37:51.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec44ef70-783c-11ef-baf6-efdc52797490 -- bash -c 'set -ex 2024-09-21T17:37:51.734 DEBUG:teuthology.orchestra.run.smithi018:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-21T17:37:51.734 DEBUG:teuthology.orchestra.run.smithi018:> ' 2024-09-21T17:37:51.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:51 smithi157 ceph-mon[49460]: pgmap v392: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-21T17:37:51.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:51 smithi157 ceph-mon[49460]: mgrmap e44: smithi018.jndyyn(active, since 10m), standbys: smithi157.oowooq 2024-09-21T17:37:53.546 INFO:teuthology.orchestra.run.smithi018.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-21T17:37:53.546 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:53 smithi018 ceph-mon[63924]: pgmap v393: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-21T17:37:53.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:53 smithi157 ceph-mon[49460]: pgmap v393: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-21T17:37:53.802 INFO:teuthology.orchestra.run.smithi018.stderr:+ [[ 7 -gt 2 ]] 2024-09-21T17:37:54.366 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:54 smithi018 ceph-mon[63924]: from='client.? 172.21.15.18:0/1433564306' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-21T17:37:54.390 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T17:37:54.400 INFO:tasks.cephadm:Teardown begin 2024-09-21T17:37:54.400 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T17:37:54.428 DEBUG:teuthology.orchestra.run.smithi157:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T17:37:54.462 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T17:37:54.462 DEBUG:teuthology.orchestra.run.smithi018:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T17:37:54.485 DEBUG:teuthology.orchestra.run.smithi157:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T17:37:54.519 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T17:37:54.520 INFO:tasks.cephadm.mon.smithi018:Stopping mon.smithi018... 2024-09-21T17:37:54.520 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018 2024-09-21T17:37:54.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:54 smithi018 systemd[1]: Stopping Ceph mon.smithi018 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:37:54.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:54 smithi157 ceph-mon[49460]: from='client.? 172.21.15.18:0/1433564306' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-21T17:37:55.054 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:54 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[63920]: 2024-09-21T17:37:54.825+0000 7f91b6057640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi018 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:37:55.054 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:54 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018[63920]: 2024-09-21T17:37:54.825+0000 7f91b6057640 -1 mon.smithi018@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T17:37:55.333 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:55 smithi018 podman[93463]: 2024-09-21 17:37:55.052650497 +0000 UTC m=+0.336925637 container died d7cfe2f8a92d77056c5a5814d0de87eecd3188e23d079e4422a3bcdea9282fa6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:37:55.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:55 smithi018 podman[93463]: 2024-09-21 17:37:55.451652622 +0000 UTC m=+0.735927751 container cleanup d7cfe2f8a92d77056c5a5814d0de87eecd3188e23d079e4422a3bcdea9282fa6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T17:37:55.653 INFO:journalctl@ceph.mon.smithi018.smithi018.stdout:Sep 21 17:37:55 smithi018 bash[93463]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi018 2024-09-21T17:37:55.857 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi018.service' 2024-09-21T17:37:55.906 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:37:55.906 INFO:tasks.cephadm.mon.smithi018:Stopped mon.smithi018 2024-09-21T17:37:55.906 INFO:tasks.cephadm.mon.smithi157:Stopping mon.smithi157... 2024-09-21T17:37:55.906 DEBUG:teuthology.orchestra.run.smithi157:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi157 2024-09-21T17:37:56.278 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:55 smithi157 systemd[1]: Stopping Ceph mon.smithi157 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:37:56.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:56 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157[49456]: 2024-09-21T17:37:56.296+0000 7fcc19b6a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi157 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:37:56.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:56 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157[49456]: 2024-09-21T17:37:56.296+0000 7fcc19b6a640 -1 mon.smithi157@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T17:37:56.778 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:56 smithi157 podman[68604]: 2024-09-21 17:37:56.480357572 +0000 UTC m=+0.381740166 container died 1c6557eec7d04ac5b25070e9e3d683f453969b02cd981f925565a600ef42e208 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-21T17:37:57.275 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:56 smithi157 podman[68604]: 2024-09-21 17:37:56.996786287 +0000 UTC m=+0.898168876 container cleanup 1c6557eec7d04ac5b25070e9e3d683f453969b02cd981f925565a600ef42e208 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:37:57.275 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:56 smithi157 bash[68604]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157 2024-09-21T17:37:57.528 INFO:journalctl@ceph.mon.smithi157.smithi157.stdout:Sep 21 17:37:57 smithi157 podman[68616]: 2024-09-21 17:37:57.273419799 +0000 UTC m=+0.792882696 container remove 1c6557eec7d04ac5b25070e9e3d683f453969b02cd981f925565a600ef42e208 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-mon-smithi157, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:37:57.571 DEBUG:teuthology.orchestra.run.smithi157:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@mon.smithi157.service' 2024-09-21T17:37:57.622 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:37:57.622 INFO:tasks.cephadm.mon.smithi157:Stopped mon.smithi157 2024-09-21T17:37:57.622 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T17:37:57.622 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.0 2024-09-21T17:37:57.903 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:37:57 smithi018 systemd[1]: Stopping Ceph osd.0 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:37:58.403 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:37:57 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[68718]: 2024-09-21T17:37:57.963+0000 7f8118c0c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:37:58.403 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:37:57 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[68718]: 2024-09-21T17:37:57.963+0000 7f8118c0c640 -1 osd.0 153 *** Got signal Terminated *** 2024-09-21T17:37:58.403 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:37:57 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0[68718]: 2024-09-21T17:37:57.963+0000 7f8118c0c640 -1 osd.0 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:38:03.818 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:03 smithi018 podman[93595]: 2024-09-21 17:38:03.53026313 +0000 UTC m=+5.700160472 container died ac5dc125292239739ae47b627670fefc6c6c9c67495a40870fd918e4e0f1a458 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T17:38:04.153 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:03 smithi018 podman[93595]: 2024-09-21 17:38:03.9142966 +0000 UTC m=+6.084193944 container cleanup ac5dc125292239739ae47b627670fefc6c6c9c67495a40870fd918e4e0f1a458 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-09-21T17:38:04.153 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:03 smithi018 bash[93595]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0 2024-09-21T17:38:04.503 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:04 smithi018 podman[93626]: 2024-09-21 17:38:04.165621389 +0000 UTC m=+0.634877049 container remove ac5dc125292239739ae47b627670fefc6c6c9c67495a40870fd918e4e0f1a458 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T17:38:04.829 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:04 smithi018 podman[93716]: 2024-09-21 17:38:04.405270884 +0000 UTC m=+0.016802944 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:38:04.829 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:04 smithi018 podman[93716]: 2024-09-21 17:38:04.533349309 +0000 UTC m=+0.144881391 container create e43788244ae25161829d72ba9f15de6106cebee4584d9197a14a4e846067918e (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T17:38:04.830 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:04 smithi018 podman[93716]: 2024-09-21 17:38:04.744491822 +0000 UTC m=+0.356023875 container init e43788244ae25161829d72ba9f15de6106cebee4584d9197a14a4e846067918e (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T17:38:04.830 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:04 smithi018 podman[93716]: 2024-09-21 17:38:04.74793906 +0000 UTC m=+0.359471113 container start e43788244ae25161829d72ba9f15de6106cebee4584d9197a14a4e846067918e (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:38:04.830 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:04 smithi018 podman[93716]: 2024-09-21 17:38:04.827728554 +0000 UTC m=+0.439260623 container attach e43788244ae25161829d72ba9f15de6106cebee4584d9197a14a4e846067918e (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T17:38:05.153 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 21 17:38:04 smithi018 podman[93716]: 2024-09-21 17:38:04.939365981 +0000 UTC m=+0.550898029 container died e43788244ae25161829d72ba9f15de6106cebee4584d9197a14a4e846067918e (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-21T17:38:05.541 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T17:38:05.585 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:38:05.585 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T17:38:05.585 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T17:38:05.585 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.1 2024-09-21T17:38:05.903 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:05 smithi018 systemd[1]: Stopping Ceph osd.1 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:38:06.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:05 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[71089]: 2024-09-21T17:38:05.917+0000 7fb89df3f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:38:06.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:05 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[71089]: 2024-09-21T17:38:05.917+0000 7fb89df3f640 -1 osd.1 153 *** Got signal Terminated *** 2024-09-21T17:38:06.403 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:05 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1[71089]: 2024-09-21T17:38:05.917+0000 7fb89df3f640 -1 osd.1 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:38:11.653 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:11 smithi018 podman[93818]: 2024-09-21 17:38:11.318906824 +0000 UTC m=+5.503163523 container died 6ff6ff17584097eb1939543d20811f0b61edd63d2ee4ca4a239c826e797db44f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T17:38:12.153 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:11 smithi018 podman[93818]: 2024-09-21 17:38:11.792872352 +0000 UTC m=+5.977129041 container cleanup 6ff6ff17584097eb1939543d20811f0b61edd63d2ee4ca4a239c826e797db44f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T17:38:12.153 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:11 smithi018 bash[93818]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1 2024-09-21T17:38:12.598 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:12 smithi018 podman[93839]: 2024-09-21 17:38:12.194273539 +0000 UTC m=+0.865709902 container remove 6ff6ff17584097eb1939543d20811f0b61edd63d2ee4ca4a239c826e797db44f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:38:12.882 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:12 smithi018 podman[93925]: 2024-09-21 17:38:12.49946587 +0000 UTC m=+0.017262747 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:38:12.882 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:12 smithi018 podman[93925]: 2024-09-21 17:38:12.653573345 +0000 UTC m=+0.171370213 container create 8b8b86af79d91c78cb1db2c4a05551426c34a61623276ca4765104b629ed86b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T17:38:12.883 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:12 smithi018 podman[93925]: 2024-09-21 17:38:12.881113131 +0000 UTC m=+0.398910015 container init 8b8b86af79d91c78cb1db2c4a05551426c34a61623276ca4765104b629ed86b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:38:13.153 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:12 smithi018 podman[93925]: 2024-09-21 17:38:12.884636347 +0000 UTC m=+0.402433223 container start 8b8b86af79d91c78cb1db2c4a05551426c34a61623276ca4765104b629ed86b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T17:38:13.153 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:12 smithi018 podman[93925]: 2024-09-21 17:38:12.964489276 +0000 UTC m=+0.482286371 container attach 8b8b86af79d91c78cb1db2c4a05551426c34a61623276ca4765104b629ed86b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T17:38:13.153 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 21 17:38:13 smithi018 podman[93925]: 2024-09-21 17:38:13.081849086 +0000 UTC m=+0.599645952 container died 8b8b86af79d91c78cb1db2c4a05551426c34a61623276ca4765104b629ed86b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:38:13.719 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T17:38:13.764 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:38:13.764 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T17:38:13.764 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T17:38:13.764 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.2 2024-09-21T17:38:14.099 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:13 smithi018 systemd[1]: Stopping Ceph osd.2 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:38:14.403 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:14 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[73552]: 2024-09-21T17:38:14.096+0000 7f46427b5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:38:14.403 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:14 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[73552]: 2024-09-21T17:38:14.096+0000 7f46427b5640 -1 osd.2 153 *** Got signal Terminated *** 2024-09-21T17:38:14.403 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:14 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2[73552]: 2024-09-21T17:38:14.096+0000 7f46427b5640 -1 osd.2 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:38:19.903 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:19 smithi018 podman[94028]: 2024-09-21 17:38:19.641721323 +0000 UTC m=+5.649496361 container died 3df4e3f0df26153c8f57f970bec2c865e4f905c85fc715ad29dd7407e3b073b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:38:20.314 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:20 smithi018 podman[94028]: 2024-09-21 17:38:20.054111196 +0000 UTC m=+6.061886224 container cleanup 3df4e3f0df26153c8f57f970bec2c865e4f905c85fc715ad29dd7407e3b073b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:38:20.314 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:20 smithi018 bash[94028]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2 2024-09-21T17:38:20.653 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:20 smithi018 podman[94045]: 2024-09-21 17:38:20.312235521 +0000 UTC m=+0.668621557 container remove 3df4e3f0df26153c8f57f970bec2c865e4f905c85fc715ad29dd7407e3b073b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True) 2024-09-21T17:38:21.014 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:20 smithi018 podman[94135]: 2024-09-21 17:38:20.665422109 +0000 UTC m=+0.016822854 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:38:21.014 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:20 smithi018 podman[94135]: 2024-09-21 17:38:20.781528936 +0000 UTC m=+0.132929720 container create de87111aa57e925a04ecaa4369b049d5754abb1aa06c601fc6c8bcc24c61a6bc (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T17:38:21.015 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:21 smithi018 podman[94135]: 2024-09-21 17:38:21.009097344 +0000 UTC m=+0.360498081 container init de87111aa57e925a04ecaa4369b049d5754abb1aa06c601fc6c8bcc24c61a6bc (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T17:38:21.015 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:21 smithi018 podman[94135]: 2024-09-21 17:38:21.012815192 +0000 UTC m=+0.364215925 container start de87111aa57e925a04ecaa4369b049d5754abb1aa06c601fc6c8bcc24c61a6bc (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T17:38:21.403 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:21 smithi018 podman[94135]: 2024-09-21 17:38:21.117747909 +0000 UTC m=+0.469148634 container attach de87111aa57e925a04ecaa4369b049d5754abb1aa06c601fc6c8bcc24c61a6bc (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:38:21.403 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 21 17:38:21 smithi018 podman[94135]: 2024-09-21 17:38:21.206802102 +0000 UTC m=+0.558202840 container died de87111aa57e925a04ecaa4369b049d5754abb1aa06c601fc6c8bcc24c61a6bc (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43) 2024-09-21T17:38:21.782 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.2.service' 2024-09-21T17:38:21.825 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:38:21.826 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T17:38:21.826 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T17:38:21.826 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.3 2024-09-21T17:38:22.152 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:21 smithi018 systemd[1]: Stopping Ceph osd.3 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:38:22.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:22 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:38:22.149+0000 7f7050f08640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:38:22.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:22 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:38:22.149+0000 7f7050f08640 -1 osd.3 153 *** Got signal Terminated *** 2024-09-21T17:38:22.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:22 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:38:22.149+0000 7f7050f08640 -1 osd.3 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:38:24.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:24 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:38:24.021+0000 7f704d51d640 -1 osd.3 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:00.867663+0000 front 2024-09-21T17:38:00.867641+0000 (oldest deadline 2024-09-21T17:38:23.767681+0000) 2024-09-21T17:38:25.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:25 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:38:25.036+0000 7f704d51d640 -1 osd.3 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:00.867663+0000 front 2024-09-21T17:38:00.867641+0000 (oldest deadline 2024-09-21T17:38:23.767681+0000) 2024-09-21T17:38:26.278 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:25 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:25.950+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:26.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:26 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:38:26.063+0000 7f704d51d640 -1 osd.3 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:00.867663+0000 front 2024-09-21T17:38:00.867641+0000 (oldest deadline 2024-09-21T17:38:23.767681+0000) 2024-09-21T17:38:27.180 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:26 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:26.986+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:27.180 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:27 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:27.082+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:27.180 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:26 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:26.899+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:27.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:27 smithi018 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3[75866]: 2024-09-21T17:38:27.107+0000 7f704d51d640 -1 osd.3 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:00.867663+0000 front 2024-09-21T17:38:00.867641+0000 (oldest deadline 2024-09-21T17:38:23.767681+0000) 2024-09-21T17:38:28.025 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:27 smithi018 podman[94254]: 2024-09-21 17:38:27.684550628 +0000 UTC m=+5.627215639 container died e7c60a62ac9e5c14edb41462112f5cf3b1b4445867afd785f8704420af92846f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T17:38:28.025 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:28 smithi018 podman[94254]: 2024-09-21 17:38:28.023699977 +0000 UTC m=+5.966364985 container cleanup e7c60a62ac9e5c14edb41462112f5cf3b1b4445867afd785f8704420af92846f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T17:38:28.271 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:27 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:27.907+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:28.272 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:28 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:28.064+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:28.272 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:27 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:27.981+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:28.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:28 smithi018 bash[94254]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3 2024-09-21T17:38:28.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:28 smithi018 podman[94270]: 2024-09-21 17:38:28.208659067 +0000 UTC m=+0.524128439 container remove e7c60a62ac9e5c14edb41462112f5cf3b1b4445867afd785f8704420af92846f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T17:38:28.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:28 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:28.269+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:28.903 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:28 smithi018 podman[94358]: 2024-09-21 17:38:28.553734504 +0000 UTC m=+0.016663461 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:38:28.903 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:28 smithi018 podman[94358]: 2024-09-21 17:38:28.650964147 +0000 UTC m=+0.113893088 container create 7a5c4a167d3af35a7da14443fc954d15ab61e3868d7cdc5d82fdbe25b79f0226 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T17:38:28.903 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:28 smithi018 podman[94358]: 2024-09-21 17:38:28.87043784 +0000 UTC m=+0.333366795 container init 7a5c4a167d3af35a7da14443fc954d15ab61e3868d7cdc5d82fdbe25b79f0226 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS) 2024-09-21T17:38:28.903 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:28 smithi018 podman[94358]: 2024-09-21 17:38:28.873794248 +0000 UTC m=+0.336723195 container start 7a5c4a167d3af35a7da14443fc954d15ab61e3868d7cdc5d82fdbe25b79f0226 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T17:38:29.278 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:28 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:28.943+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:29.278 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:29 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:29.061+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:29.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:29 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:29.011+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:29.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:28 smithi018 podman[94358]: 2024-09-21 17:38:28.978823869 +0000 UTC m=+0.441752825 container attach 7a5c4a167d3af35a7da14443fc954d15ab61e3868d7cdc5d82fdbe25b79f0226 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T17:38:29.403 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 21 17:38:29 smithi018 podman[94358]: 2024-09-21 17:38:29.07199376 +0000 UTC m=+0.534922702 container died 7a5c4a167d3af35a7da14443fc954d15ab61e3868d7cdc5d82fdbe25b79f0226 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T17:38:29.732 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T17:38:29.772 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:38:29.773 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T17:38:29.773 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T17:38:29.773 DEBUG:teuthology.orchestra.run.smithi157:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.4 2024-09-21T17:38:29.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:29 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:29.311+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:30.067 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:29 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:29.938+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:30.068 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:29 smithi157 systemd[1]: Stopping Ceph osd.4 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:38:30.068 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:29 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:29.973+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:30.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:30.089+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:30.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:30.270+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:30.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:30.065+0000 7efdbbc2a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:38:30.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:30.065+0000 7efdbbc2a640 -1 osd.4 153 *** Got signal Terminated *** 2024-09-21T17:38:30.529 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:30.065+0000 7efdbbc2a640 -1 osd.4 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:38:31.243 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:31 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:31.007+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:31.243 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:30 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:30.956+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:31.243 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:31 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:31.134+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:31.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:31 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:31.240+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:32.218 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:32 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:32.125+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:32.219 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:31 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:31.989+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:32.219 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:31 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:31.931+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:32.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:32 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:32.216+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:32.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:32 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:32.216+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:33.239 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:33 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:33.126+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:33.239 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:33 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:33.039+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:33.239 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:33 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:33.039+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.969462+0000 front 2024-09-21T17:38:06.969509+0000 (oldest deadline 2024-09-21T17:38:32.269324+0000) 2024-09-21T17:38:33.239 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:32 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:32.911+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:33.239 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:32 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:32.911+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:33.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:33 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:33.236+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:33.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:33 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:33.236+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:34.206 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:33 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:33.941+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:34.206 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:33 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:33.941+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:34.206 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:34.106+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:34.206 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:34.106+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:34.206 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:34.005+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:34.206 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:34.005+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.969462+0000 front 2024-09-21T17:38:06.969509+0000 (oldest deadline 2024-09-21T17:38:32.269324+0000) 2024-09-21T17:38:34.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:34.203+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:34.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:34.203+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:35.251 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:35 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:35.122+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:35.251 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:35 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:35.122+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:35.252 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:34.995+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.468317+0000 front 2024-09-21T17:38:02.468206+0000 (oldest deadline 2024-09-21T17:38:25.368052+0000) 2024-09-21T17:38:35.252 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4[53346]: 2024-09-21T17:38:34.995+0000 7efdb4237640 -1 osd.4 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.969462+0000 front 2024-09-21T17:38:06.969509+0000 (oldest deadline 2024-09-21T17:38:32.269324+0000) 2024-09-21T17:38:35.252 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:34.969+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:35.252 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:34 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:34.969+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:35.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:35 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:35.249+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:35.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:35 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:35.249+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:35.894 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:35 smithi157 podman[68770]: 2024-09-21 17:38:35.556525358 +0000 UTC m=+5.592838896 container died 2552dce6837e1a43e8deeb373f2c8a23e56d667d3825970144cdc09fe821ad94 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:38:36.244 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:36.101+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:36.244 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:36.101+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:36.245 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:36 smithi157 podman[68770]: 2024-09-21 17:38:36.082630007 +0000 UTC m=+6.118943580 container cleanup 2552dce6837e1a43e8deeb373f2c8a23e56d667d3825970144cdc09fe821ad94 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:38:36.245 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:36 smithi157 bash[68770]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4 2024-09-21T17:38:36.245 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:35 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:35.981+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:36.245 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:35 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:35.981+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:36.528 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:36 smithi157 podman[68785]: 2024-09-21 17:38:36.29275696 +0000 UTC m=+0.729508168 container remove 2552dce6837e1a43e8deeb373f2c8a23e56d667d3825970144cdc09fe821ad94 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T17:38:36.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:36.242+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:36.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:36.242+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:36.998 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:36 smithi157 podman[68874]: 2024-09-21 17:38:36.609553031 +0000 UTC m=+0.016026663 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:38:36.999 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:36 smithi157 podman[68874]: 2024-09-21 17:38:36.760278105 +0000 UTC m=+0.166751738 container create 938d64aaef1ce1a182b4826069b20941d88e5f0a5dc10aadcca733a79970bef0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T17:38:36.999 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:36.950+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:36.999 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:36 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:36.950+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:37.278 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:37.075+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:37.278 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:37.075+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:37.279 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:37.244+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:37.279 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:37.244+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:37.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:36 smithi157 podman[68874]: 2024-09-21 17:38:36.996714219 +0000 UTC m=+0.403187867 container init 938d64aaef1ce1a182b4826069b20941d88e5f0a5dc10aadcca733a79970bef0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T17:38:37.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:37 smithi157 podman[68874]: 2024-09-21 17:38:37.000646342 +0000 UTC m=+0.407119997 container start 938d64aaef1ce1a182b4826069b20941d88e5f0a5dc10aadcca733a79970bef0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T17:38:37.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:37 smithi157 podman[68874]: 2024-09-21 17:38:37.08787441 +0000 UTC m=+0.494348053 container attach 938d64aaef1ce1a182b4826069b20941d88e5f0a5dc10aadcca733a79970bef0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T17:38:37.279 INFO:journalctl@ceph.osd.4.smithi157.stdout:Sep 21 17:38:37 smithi157 podman[68874]: 2024-09-21 17:38:37.19204422 +0000 UTC m=+0.598517853 container died 938d64aaef1ce1a182b4826069b20941d88e5f0a5dc10aadcca733a79970bef0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T17:38:37.950 DEBUG:teuthology.orchestra.run.smithi157:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T17:38:38.029 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:38:38.029 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T17:38:38.029 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T17:38:38.029 DEBUG:teuthology.orchestra.run.smithi157:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.5 2024-09-21T17:38:38.257 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:38.095+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:38.258 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:38.095+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:38.258 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:37.981+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:38.258 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:37 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:37.981+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:38.258 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:38 smithi157 systemd[1]: Stopping Ceph osd.5 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:38:38.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:38.411+0000 7f4639f4f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:38:38.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:38.411+0000 7f4639f4f640 -1 osd.5 153 *** Got signal Terminated *** 2024-09-21T17:38:38.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:38.411+0000 7f4639f4f640 -1 osd.5 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:38:38.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:38.255+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:38.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:38.255+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:39.278 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:38.940+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:39.278 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:38 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:38.940+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:39.278 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:39 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:39.102+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:39.278 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:39 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:39.102+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:39.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:39 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:39.285+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:39.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:39 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:39.285+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:40.257 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:39 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:39.918+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:40.257 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:39 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:39.918+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:40.257 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:39 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:39.918+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.760573+0000 front 2024-09-21T17:38:13.760654+0000 (oldest deadline 2024-09-21T17:38:39.660556+0000) 2024-09-21T17:38:40.258 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:40 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:40.056+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:40.258 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:40 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:40.056+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:40.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:40 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:40.255+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:40.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:40 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:40.255+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:41.255 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:40 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:40.910+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:41.255 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:40 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:40.910+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:41.255 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:40 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:40.910+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.760573+0000 front 2024-09-21T17:38:13.760654+0000 (oldest deadline 2024-09-21T17:38:39.660556+0000) 2024-09-21T17:38:41.255 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:41 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:41.043+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:41.255 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:41 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:41.043+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:41.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:41 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:41.252+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:41.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:41 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:41.252+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:42.231 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:41 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:41.918+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:42.231 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:41 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:41.918+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:42.231 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:41 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:41.918+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.760573+0000 front 2024-09-21T17:38:13.760654+0000 (oldest deadline 2024-09-21T17:38:39.660556+0000) 2024-09-21T17:38:42.231 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:42 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:42.053+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:42.231 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:42 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:42.053+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:42.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:42 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:42.228+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:42.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:42 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:42.228+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:43.238 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:42 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:42.871+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.059332+0000 front 2024-09-21T17:38:02.059258+0000 (oldest deadline 2024-09-21T17:38:26.159059+0000) 2024-09-21T17:38:43.239 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:42 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:42.871+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:06.159811+0000 front 2024-09-21T17:38:06.159799+0000 (oldest deadline 2024-09-21T17:38:32.059693+0000) 2024-09-21T17:38:43.239 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:42 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5[55374]: 2024-09-21T17:38:42.871+0000 7f4636564640 -1 osd.5 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.760573+0000 front 2024-09-21T17:38:13.760654+0000 (oldest deadline 2024-09-21T17:38:39.660556+0000) 2024-09-21T17:38:43.239 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:43 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:43.018+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:43.239 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:43 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:43.018+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:43.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:43 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:43.236+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:43.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:43 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:43.236+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:44.081 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:43 smithi157 podman[68988]: 2024-09-21 17:38:43.794684116 +0000 UTC m=+5.513413548 container died abad654e4600e3db95fc79720ae5049cfb7acc4e97867da362261962b108e478 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T17:38:44.081 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:44 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:44.027+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:44.082 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:44 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:44.027+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:44.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:44 smithi157 podman[68988]: 2024-09-21 17:38:44.295655054 +0000 UTC m=+6.014384545 container cleanup abad654e4600e3db95fc79720ae5049cfb7acc4e97867da362261962b108e478 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T17:38:44.528 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:44 smithi157 bash[68988]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5 2024-09-21T17:38:44.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:44 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:44.210+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:44.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:44 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:44.210+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:44.898 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:44 smithi157 podman[69021]: 2024-09-21 17:38:44.572867693 +0000 UTC m=+0.776757066 container remove abad654e4600e3db95fc79720ae5049cfb7acc4e97867da362261962b108e478 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T17:38:45.201 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:44 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:44.978+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:45.202 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:44 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:44.978+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:45.202 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:44 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:44.978+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.391374+0000 front 2024-09-21T17:38:13.391311+0000 (oldest deadline 2024-09-21T17:38:44.591679+0000) 2024-09-21T17:38:45.202 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:44 smithi157 podman[69111]: 2024-09-21 17:38:44.798934082 +0000 UTC m=+0.015827750 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:38:45.202 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:44 smithi157 podman[69111]: 2024-09-21 17:38:44.93155346 +0000 UTC m=+0.148447155 container create 66e70b935bb4ecc17a9e27edd1646600c53efffe20dc8127b82f86e3377198e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:38:45.202 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:45 smithi157 podman[69111]: 2024-09-21 17:38:45.200257758 +0000 UTC m=+0.417151416 container init 66e70b935bb4ecc17a9e27edd1646600c53efffe20dc8127b82f86e3377198e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:38:45.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:45 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:45.237+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:45.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:45 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:45.237+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:45.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:45 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:45.237+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:45.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:45 smithi157 podman[69111]: 2024-09-21 17:38:45.203693114 +0000 UTC m=+0.420586779 container start 66e70b935bb4ecc17a9e27edd1646600c53efffe20dc8127b82f86e3377198e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T17:38:45.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:45 smithi157 podman[69111]: 2024-09-21 17:38:45.317617441 +0000 UTC m=+0.534511114 container attach 66e70b935bb4ecc17a9e27edd1646600c53efffe20dc8127b82f86e3377198e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T17:38:45.529 INFO:journalctl@ceph.osd.5.smithi157.stdout:Sep 21 17:38:45 smithi157 podman[69111]: 2024-09-21 17:38:45.393263035 +0000 UTC m=+0.610156688 container died 66e70b935bb4ecc17a9e27edd1646600c53efffe20dc8127b82f86e3377198e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T17:38:46.052 DEBUG:teuthology.orchestra.run.smithi157:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T17:38:46.092 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:38:46.093 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T17:38:46.093 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T17:38:46.093 DEBUG:teuthology.orchestra.run.smithi157:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.6 2024-09-21T17:38:46.278 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:46.231+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:46.278 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:46.231+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:46.278 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:46.231+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:46.279 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:45 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:45.998+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:46.279 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:45 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:45.998+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:46.279 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:45 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:45.998+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.391374+0000 front 2024-09-21T17:38:13.391311+0000 (oldest deadline 2024-09-21T17:38:44.591679+0000) 2024-09-21T17:38:46.279 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:46 smithi157 systemd[1]: Stopping Ceph osd.6 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:38:46.778 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:46.432+0000 7f5fff17c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:38:46.779 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:46.432+0000 7f5fff17c640 -1 osd.6 153 *** Got signal Terminated *** 2024-09-21T17:38:46.779 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:46.432+0000 7f5fff17c640 -1 osd.6 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:38:47.278 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:46.989+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:47.279 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:46.989+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:47.279 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:46 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:46.989+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.391374+0000 front 2024-09-21T17:38:13.391311+0000 (oldest deadline 2024-09-21T17:38:44.591679+0000) 2024-09-21T17:38:47.279 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:47 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:47.209+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:47.279 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:47 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:47.209+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:47.279 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:47 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:47.209+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:48.207 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:48 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:48.035+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:48.207 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:48 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:48.035+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:48.207 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:48 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:48.035+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.391374+0000 front 2024-09-21T17:38:13.391311+0000 (oldest deadline 2024-09-21T17:38:44.591679+0000) 2024-09-21T17:38:48.495 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:48 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:48.205+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:48.496 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:48 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:48.205+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:48.496 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:48 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:48.205+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:48.496 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:48 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:48.205+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:49.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:49 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:49.074+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:49.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:49 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:49.074+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:49.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:49 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:49.074+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.391374+0000 front 2024-09-21T17:38:13.391311+0000 (oldest deadline 2024-09-21T17:38:44.591679+0000) 2024-09-21T17:38:49.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:49 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:49.250+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:49.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:49 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:49.250+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:49.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:49 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:49.250+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:49.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:49 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:49.250+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:50.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:50 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:50.078+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:50.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:50 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:50.078+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:50.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:50 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:50.078+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.391374+0000 front 2024-09-21T17:38:13.391311+0000 (oldest deadline 2024-09-21T17:38:44.591679+0000) 2024-09-21T17:38:50.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:50 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:50.267+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:50.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:50 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:50.267+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:50.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:50 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:50.267+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:50.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:50 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:50.267+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:51.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:51 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:51.074+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.290113+0000 front 2024-09-21T17:38:02.290269+0000 (oldest deadline 2024-09-21T17:38:26.390029+0000) 2024-09-21T17:38:51.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:51 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:51.074+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:08.090947+0000 front 2024-09-21T17:38:08.090957+0000 (oldest deadline 2024-09-21T17:38:33.390821+0000) 2024-09-21T17:38:51.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:51 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:51.074+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:13.391374+0000 front 2024-09-21T17:38:13.391311+0000 (oldest deadline 2024-09-21T17:38:44.591679+0000) 2024-09-21T17:38:51.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:51 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6[57379]: 2024-09-21T17:38:51.074+0000 7f5ffb791640 -1 osd.6 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:24.592206+0000 front 2024-09-21T17:38:24.592255+0000 (oldest deadline 2024-09-21T17:38:50.492126+0000) 2024-09-21T17:38:51.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:51 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:51.248+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:51.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:51 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:51.248+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:51.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:51 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:51.248+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:51.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:51 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:51.248+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:52.109 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:51 smithi157 podman[69215]: 2024-09-21 17:38:51.84853055 +0000 UTC m=+5.527648357 container died fd2e6eb21352c5bc7b7470d37bb5d51eb3372ad690b728112c9ca5cc2e6ac98f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T17:38:52.493 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:52 smithi157 podman[69215]: 2024-09-21 17:38:52.290591978 +0000 UTC m=+5.969709845 container cleanup fd2e6eb21352c5bc7b7470d37bb5d51eb3372ad690b728112c9ca5cc2e6ac98f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T17:38:52.494 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:52 smithi157 bash[69215]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6 2024-09-21T17:38:52.494 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:52.244+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:52.494 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:52.244+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:52.494 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:52.244+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:52.494 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:52 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:52.244+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:52.778 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:52 smithi157 podman[69230]: 2024-09-21 17:38:52.492084825 +0000 UTC m=+0.640044116 container remove fd2e6eb21352c5bc7b7470d37bb5d51eb3372ad690b728112c9ca5cc2e6ac98f (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:38:53.188 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:52 smithi157 podman[69320]: 2024-09-21 17:38:52.806477892 +0000 UTC m=+0.016163300 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:38:53.189 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:52 smithi157 podman[69320]: 2024-09-21 17:38:52.926065972 +0000 UTC m=+0.135751379 container create f2290fbc15d26df143e01fee982221d3728d52996340544d6eb6e5c05d983065 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T17:38:53.189 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:53 smithi157 podman[69320]: 2024-09-21 17:38:53.187211009 +0000 UTC m=+0.396896422 container init f2290fbc15d26df143e01fee982221d3728d52996340544d6eb6e5c05d983065 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T17:38:53.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:53 smithi157 podman[69320]: 2024-09-21 17:38:53.190554095 +0000 UTC m=+0.400239504 container start f2290fbc15d26df143e01fee982221d3728d52996340544d6eb6e5c05d983065 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T17:38:53.528 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:53 smithi157 podman[69320]: 2024-09-21 17:38:53.278784397 +0000 UTC m=+0.488469805 container attach f2290fbc15d26df143e01fee982221d3728d52996340544d6eb6e5c05d983065 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2) 2024-09-21T17:38:53.529 INFO:journalctl@ceph.osd.6.smithi157.stdout:Sep 21 17:38:53 smithi157 podman[69320]: 2024-09-21 17:38:53.377151163 +0000 UTC m=+0.586836579 container died f2290fbc15d26df143e01fee982221d3728d52996340544d6eb6e5c05d983065 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T17:38:53.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:53 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:53.263+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:53.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:53 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:53.263+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:53.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:53 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:53.263+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:53.529 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:53 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:53.263+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:54.025 DEBUG:teuthology.orchestra.run.smithi157:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.6.service' 2024-09-21T17:38:54.063 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:38:54.063 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T17:38:54.063 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T17:38:54.063 DEBUG:teuthology.orchestra.run.smithi157:> sudo systemctl stop ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.7 2024-09-21T17:38:54.278 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:54 smithi157 systemd[1]: Stopping Ceph osd.7 for ec44ef70-783c-11ef-baf6-efdc52797490... 2024-09-21T17:38:54.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:54 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:54.277+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:54.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:54 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:54.277+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:54.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:54 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:54.277+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:54.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:54 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:54.277+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:54.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:54 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:54.419+0000 7f3614c5f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T17:38:54.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:54 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:54.419+0000 7f3614c5f640 -1 osd.7 153 *** Got signal Terminated *** 2024-09-21T17:38:54.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:54 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:54.419+0000 7f3614c5f640 -1 osd.7 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T17:38:55.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:55 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:55.237+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:55.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:55 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:55.237+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:55.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:55 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:55.237+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:55.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:55 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:55.237+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:56.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:56 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:56.213+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:56.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:56 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:56.213+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:56.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:56 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:56.213+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:56.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:56 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:56.213+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:57.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:57 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:57.255+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:57.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:57 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:57.255+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:57.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:57 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:57.255+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:57.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:57 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:57.255+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:58.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:58.274+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:58.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:58.274+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:58.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:58.274+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:58.528 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:58 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:58.274+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:59.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:59 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:59.275+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2024-09-21T17:38:02.236163+0000 front 2024-09-21T17:38:02.236018+0000 (oldest deadline 2024-09-21T17:38:27.536021+0000) 2024-09-21T17:38:59.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:59 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:59.275+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2024-09-21T17:38:07.536685+0000 front 2024-09-21T17:38:07.536701+0000 (oldest deadline 2024-09-21T17:38:31.636537+0000) 2024-09-21T17:38:59.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:59 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:59.275+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2024-09-21T17:38:15.637811+0000 front 2024-09-21T17:38:15.637740+0000 (oldest deadline 2024-09-21T17:38:45.038012+0000) 2024-09-21T17:38:59.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:59 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:59.275+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2024-09-21T17:38:25.038620+0000 front 2024-09-21T17:38:25.038575+0000 (oldest deadline 2024-09-21T17:38:47.938402+0000) 2024-09-21T17:38:59.779 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:59 smithi157 ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7[59400]: 2024-09-21T17:38:59.275+0000 7f3611274640 -1 osd.7 153 heartbeat_check: no reply from 172.21.15.157:6804 osd.4 since back 2024-09-21T17:38:34.839990+0000 front 2024-09-21T17:38:34.840119+0000 (oldest deadline 2024-09-21T17:38:58.339977+0000) 2024-09-21T17:39:00.278 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:38:59 smithi157 podman[69422]: 2024-09-21 17:38:59.998442587 +0000 UTC m=+5.710262091 container died 85bf07d224b1d30fa641ebf8d944196a3f589a38f717cddab35c02277d912848 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T17:39:00.642 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:00 smithi157 podman[69422]: 2024-09-21 17:39:00.406351356 +0000 UTC m=+6.118170860 container cleanup 85bf07d224b1d30fa641ebf8d944196a3f589a38f717cddab35c02277d912848 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2) 2024-09-21T17:39:00.643 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:00 smithi157 bash[69422]: ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7 2024-09-21T17:39:01.028 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:00 smithi157 podman[69450]: 2024-09-21 17:39:00.640991643 +0000 UTC m=+0.641469047 container remove 85bf07d224b1d30fa641ebf8d944196a3f589a38f717cddab35c02277d912848 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T17:39:01.362 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:01 smithi157 podman[69541]: 2024-09-21 17:39:00.976010559 +0000 UTC m=+0.015764940 image pull 3fa9090ef91d4e87f14d21d1361976a766f2908d39879e744bc3fd80d5762a25 quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5 2024-09-21T17:39:01.362 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:01 smithi157 podman[69541]: 2024-09-21 17:39:01.116824883 +0000 UTC m=+0.156579297 container create 093b64a773c61c46977d2e4774588f5f6a6aca889090fae09c2e0a041a9ceb63 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T17:39:01.362 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:01 smithi157 podman[69541]: 2024-09-21 17:39:01.361128809 +0000 UTC m=+0.400883187 container init 093b64a773c61c46977d2e4774588f5f6a6aca889090fae09c2e0a041a9ceb63 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:39:01.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:01 smithi157 podman[69541]: 2024-09-21 17:39:01.364487539 +0000 UTC m=+0.404241909 container start 093b64a773c61c46977d2e4774588f5f6a6aca889090fae09c2e0a041a9ceb63 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:39:01.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:01 smithi157 podman[69541]: 2024-09-21 17:39:01.444309882 +0000 UTC m=+0.484064271 container attach 093b64a773c61c46977d2e4774588f5f6a6aca889090fae09c2e0a041a9ceb63 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T17:39:01.778 INFO:journalctl@ceph.osd.7.smithi157.stdout:Sep 21 17:39:01 smithi157 podman[69541]: 2024-09-21 17:39:01.554132645 +0000 UTC m=+0.593887024 container died 093b64a773c61c46977d2e4774588f5f6a6aca889090fae09c2e0a041a9ceb63 (image=quay.ceph.io/ceph-ci/ceph@sha256:b7e696d4014dc129bd30a3126ccae3c43a4a3060f7a811b44f457f7ec9f7f5d5, name=ceph-ec44ef70-783c-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-21-1014-0ecbb43, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T17:39:02.167 DEBUG:teuthology.orchestra.run.smithi157:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec44ef70-783c-11ef-baf6-efdc52797490@osd.7.service' 2024-09-21T17:39:02.205 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T17:39:02.205 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T17:39:02.206 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ec44ef70-783c-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T17:42:01.821 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ec44ef70-783c-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T17:42:20.821 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T17:42:20.856 DEBUG:teuthology.orchestra.run.smithi157:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T17:42:20.883 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T17:42:20.885 DEBUG:teuthology.misc:Transferring archived files from smithi018:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048/remote/smithi018/crash 2024-09-21T17:42:20.886 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/crash -- . 2024-09-21T17:42:20.936 INFO:teuthology.orchestra.run.smithi018.stderr:tar: /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T17:42:20.936 INFO:teuthology.orchestra.run.smithi018.stderr:tar: Error is not recoverable: exiting now 2024-09-21T17:42:20.938 DEBUG:teuthology.misc:Transferring archived files from smithi157:/var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048/remote/smithi157/crash 2024-09-21T17:42:20.939 DEBUG:teuthology.orchestra.run.smithi157:> sudo tar c -f - -C /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/crash -- . 2024-09-21T17:42:20.974 INFO:teuthology.orchestra.run.smithi157.stderr:tar: /var/lib/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T17:42:20.975 INFO:teuthology.orchestra.run.smithi157.stderr:tar: Error is not recoverable: exiting now 2024-09-21T17:42:20.975 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T17:42:20.976 DEBUG:teuthology.orchestra.run.smithi018:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-21T17:42:21.009 INFO:tasks.cephadm:Compressing logs... 2024-09-21T17:42:21.010 DEBUG:teuthology.orchestra.run.smithi018:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T17:42:21.053 DEBUG:teuthology.orchestra.run.smithi157:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T17:42:21.077 INFO:teuthology.orchestra.run.smithi157.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi157.iijaxo.log 2024-09-21T17:42:21.077 INFO:teuthology.orchestra.run.smithi157.stderr:: No such file or directory 2024-09-21T17:42:21.077 INFO:teuthology.orchestra.run.smithi157.stderr:gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.7.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.0.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:: No such file or directory 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.1.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi018.wzitfj.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi018.wzitfj.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mgr.smithi018.jndyyn.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr: 88.9% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi018.wzitfj.log.gz 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mgr.smithi018.jndyyn.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mon.smithi018.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.2.log 2024-09-21T17:42:21.084 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mon.smithi018.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi018.stderr: 85.7% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.3.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi157.iijaxo.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.4.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr: 88.3% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi157.iijaxo.log.gz 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mgr.smithi157.oowooq.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mgr.smithi157.oowooq.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.6.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.5.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.log 2024-09-21T17:42:21.085 INFO:teuthology.orchestra.run.smithi157.stderr: 85.7% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T17:42:21.086 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.5.log: 91.6% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T17:42:21.087 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.log: 91.4% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T17:42:21.087 INFO:teuthology.orchestra.run.smithi018.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T17:42:21.087 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.3.log: 87.1% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T17:42:21.092 INFO:teuthology.orchestra.run.smithi157.stderr: 91.2% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T17:42:21.092 INFO:teuthology.orchestra.run.smithi157.stderr:gzip -5 --verbose -- /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mon.smithi157.log 2024-09-21T17:42:21.093 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T17:42:21.094 INFO:teuthology.orchestra.run.smithi018.stderr:/var/log/ceph/cephadm.log: 91.1% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T17:42:21.096 INFO:teuthology.orchestra.run.smithi157.stderr:/var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mon.smithi157.log: /var/log/ceph/cephadm.log: 87.1% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T17:42:21.100 INFO:teuthology.orchestra.run.smithi157.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T17:42:21.128 INFO:teuthology.orchestra.run.smithi018.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T17:42:21.149 INFO:teuthology.orchestra.run.smithi157.stderr: 89.8% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mgr.smithi157.oowooq.log.gz 2024-09-21T17:42:21.427 INFO:teuthology.orchestra.run.smithi018.stderr: 88.7% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mgr.smithi018.jndyyn.log.gz 2024-09-21T17:42:21.547 INFO:teuthology.orchestra.run.smithi157.stderr: 92.6% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mon.smithi157.log.gz 2024-09-21T17:42:22.401 INFO:teuthology.orchestra.run.smithi018.stderr: 91.8% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-mon.smithi018.log.gz 2024-09-21T17:42:34.198 INFO:teuthology.orchestra.run.smithi018.stderr: 94.5% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.2.log.gz 2024-09-21T17:42:34.660 INFO:teuthology.orchestra.run.smithi157.stderr: 93.9% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.4.log.gz 2024-09-21T17:42:35.261 INFO:teuthology.orchestra.run.smithi157.stderr: 94.3% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.6.log.gz 2024-09-21T17:42:36.420 INFO:teuthology.orchestra.run.smithi018.stderr: 94.1% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.0.log.gz 2024-09-21T17:42:38.584 INFO:teuthology.orchestra.run.smithi018.stderr: 94.1% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.3.log.gz 2024-09-21T17:42:38.926 INFO:teuthology.orchestra.run.smithi018.stderr: 94.2% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.1.log.gz 2024-09-21T17:42:38.928 INFO:teuthology.orchestra.run.smithi018.stderr: 2024-09-21T17:42:38.928 INFO:teuthology.orchestra.run.smithi018.stderr:real 0m17.862s 2024-09-21T17:42:38.928 INFO:teuthology.orchestra.run.smithi018.stderr:user 1m3.129s 2024-09-21T17:42:38.928 INFO:teuthology.orchestra.run.smithi018.stderr:sys 0m2.306s 2024-09-21T17:42:39.261 INFO:teuthology.orchestra.run.smithi157.stderr: 94.2% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.5.log.gz 2024-09-21T17:42:42.018 INFO:teuthology.orchestra.run.smithi157.stderr: 94.1% -- replaced with /var/log/ceph/ec44ef70-783c-11ef-baf6-efdc52797490/ceph-osd.7.log.gz 2024-09-21T17:42:42.020 INFO:teuthology.orchestra.run.smithi157.stderr: 2024-09-21T17:42:42.020 INFO:teuthology.orchestra.run.smithi157.stderr:real 0m20.953s 2024-09-21T17:42:42.020 INFO:teuthology.orchestra.run.smithi157.stderr:user 1m4.725s 2024-09-21T17:42:42.020 INFO:teuthology.orchestra.run.smithi157.stderr:sys 0m2.602s 2024-09-21T17:42:42.020 INFO:tasks.cephadm:Archiving logs... 2024-09-21T17:42:42.021 DEBUG:teuthology.misc:Transferring archived files from smithi018:/var/log/ceph to /home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048/remote/smithi018/log 2024-09-21T17:42:42.022 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T17:42:50.156 DEBUG:teuthology.misc:Transferring archived files from smithi157:/var/log/ceph to /home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048/remote/smithi157/log 2024-09-21T17:42:50.157 DEBUG:teuthology.orchestra.run.smithi157:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T17:42:58.888 INFO:tasks.cephadm:Removing cluster... 2024-09-21T17:42:58.888 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ec44ef70-783c-11ef-baf6-efdc52797490 --force 2024-09-21T17:42:59.259 DEBUG:teuthology.orchestra.run.smithi157:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ec44ef70-783c-11ef-baf6-efdc52797490 --force 2024-09-21T17:42:59.633 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T17:42:59.634 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T17:42:59.650 DEBUG:teuthology.orchestra.run.smithi157:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T17:42:59.665 INFO:tasks.cephadm:Teardown complete 2024-09-21T17:42:59.665 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T17:42:59.678 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T17:42:59.679 DEBUG:teuthology.orchestra.run.smithi018:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T17:42:59.692 DEBUG:teuthology.orchestra.run.smithi157:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T17:42:59.704 INFO:teuthology.orchestra.run.smithi018.stderr:bash: line 1: ntpq: command not found 2024-09-21T17:42:59.709 INFO:teuthology.orchestra.run.smithi018.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T17:42:59.710 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================== 2024-09-21T17:42:59.710 INFO:teuthology.orchestra.run.smithi018.stdout:^* hv01.front.sepia.ceph.com 3 6 377 38 -279us[ -233us] +/- 38ms 2024-09-21T17:42:59.710 INFO:teuthology.orchestra.run.smithi018.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 37 +905us[ +905us] +/- 69ms 2024-09-21T17:42:59.710 INFO:teuthology.orchestra.run.smithi018.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 38 -266us[ -220us] +/- 68ms 2024-09-21T17:42:59.710 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:42:59.719 INFO:teuthology.orchestra.run.smithi157.stderr:bash: line 1: ntpq: command not found 2024-09-21T17:42:59.724 INFO:teuthology.orchestra.run.smithi157.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T17:42:59.724 INFO:teuthology.orchestra.run.smithi157.stdout:=============================================================================== 2024-09-21T17:42:59.724 INFO:teuthology.orchestra.run.smithi157.stdout:^* hv01.front.sepia.ceph.com 3 6 377 38 -299us[ -266us] +/- 38ms 2024-09-21T17:42:59.724 INFO:teuthology.orchestra.run.smithi157.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 40 +884us[ +918us] +/- 69ms 2024-09-21T17:42:59.724 INFO:teuthology.orchestra.run.smithi157.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 38 -288us[ -254us] +/- 68ms 2024-09-21T17:42:59.724 INFO:teuthology.orchestra.run.smithi157.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T17:42:59.725 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T17:42:59.733 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T17:42:59.734 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T17:42:59.772 DEBUG:teuthology.orchestra.run.smithi018:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T17:42:59.850 DEBUG:teuthology.orchestra.run.smithi157:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T17:42:59.922 DEBUG:teuthology.orchestra.run.smithi018:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T17:42:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T17:42:59.951 DEBUG:teuthology.orchestra.run.smithi157:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T17:42:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T17:42:59.979 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T17:42:59.988 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T17:43:00.005 INFO:teuthology.task.internal:Duration was 1952.099955 seconds 2024-09-21T17:43:00.006 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T17:43:00.044 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T17:43:00.045 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T17:43:00.047 DEBUG:teuthology.orchestra.run.smithi157:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T17:43:00.085 INFO:teuthology.orchestra.run.smithi018.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T17:43:00.086 INFO:teuthology.orchestra.run.smithi157.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T17:43:00.446 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T17:43:00.446 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi018.front.sepia.ceph.com 2024-09-21T17:43:00.446 DEBUG:teuthology.orchestra.run.smithi018:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T17:43:00.508 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi157.front.sepia.ceph.com 2024-09-21T17:43:00.508 DEBUG:teuthology.orchestra.run.smithi157:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T17:43:00.535 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T17:43:00.536 DEBUG:teuthology.orchestra.run.smithi018:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T17:43:00.551 DEBUG:teuthology.orchestra.run.smithi157:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T17:43:00.602 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T17:43:00.603 DEBUG:teuthology.orchestra.run.smithi018:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T17:43:00.619 DEBUG:teuthology.orchestra.run.smithi157:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T17:43:15.054 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T17:43:15.065 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T17:43:15.066 DEBUG:teuthology.orchestra.run.smithi018:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T17:43:15.092 DEBUG:teuthology.orchestra.run.smithi157:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T17:43:15.123 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T17:43:15.133 DEBUG:teuthology.orchestra.run.smithi018:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T17:43:15.136 DEBUG:teuthology.orchestra.run.smithi157:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T17:43:15.157 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern = core 2024-09-21T17:43:15.190 INFO:teuthology.orchestra.run.smithi157.stdout:kernel.core_pattern = core 2024-09-21T17:43:15.348 DEBUG:teuthology.orchestra.run.smithi018:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T17:43:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T17:43:15.363 DEBUG:teuthology.orchestra.run.smithi157:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T17:43:15.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T17:43:15.406 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T17:43:15.424 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T17:43:15.424 DEBUG:teuthology.misc:Transferring archived files from smithi018:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048/remote/smithi018 2024-09-21T17:43:15.425 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T17:43:15.486 DEBUG:teuthology.misc:Transferring archived files from smithi157:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-09-21_16:57:11-orch:cephadm-wip-adk-testing-2024-09-21-1014-distro-default-smithi/7914048/remote/smithi157 2024-09-21T17:43:15.486 DEBUG:teuthology.orchestra.run.smithi157:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T17:43:15.527 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T17:43:15.528 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T17:43:15.530 DEBUG:teuthology.orchestra.run.smithi157:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T17:43:15.578 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T17:43:15.590 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T17:43:15.590 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T17:43:15.614 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T17:43:15.615 DEBUG:teuthology.orchestra.run.smithi018:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T17:43:15.617 DEBUG:teuthology.orchestra.run.smithi157:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T17:43:15.630 INFO:teuthology.orchestra.run.smithi018.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 17:43 /home/ubuntu/cephtest 2024-09-21T17:43:15.635 INFO:teuthology.orchestra.run.smithi157.stdout: 266054 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 17:43 /home/ubuntu/cephtest 2024-09-21T17:43:15.636 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T17:43:15.647 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T17:43:15.692 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1952.0999550819397 owner: scheduled_adking@teuthology success: true 2024-09-21T17:43:15.692 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T17:43:15.785 INFO:teuthology.run:pass