2024-04-12T21:29:27.300 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-12T21:29:27.300 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --name teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi --description orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --verbose --archive /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541 -- /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541/orig.config.yaml 2024-04-12T21:29:27.337 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-12T21:29:27.417 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7654541' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: 98423fadeecbba811088c3136f8295dae6dfe870 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 2723 sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a sleep_before_teardown: 0 subset: 24/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_98423fadeecbba811088c3136f8295dae6dfe870/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 98423fadeecbba811088c3136f8295dae6dfe870 targets: smithi123.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGmn6Phy9Qyz8bcmSfXEWGoGASyjs+Z70dhNTVJ+VNRDnDca15Dme2F1oo44XBwBm3Bws9Latr+JelBnYJF8Mfk= smithi177.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP5navxQ0Zgey4yHcCk2usMWx4nnxrcWaXPPRSetGO/ESja/FSz912f94o5bW8g5Nze5LHJnfSHoODgUvdLqR/w= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-12_21:08:02 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-12T21:29:27.417 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_98423fadeecbba811088c3136f8295dae6dfe870/qa; will attempt to use it 2024-04-12T21:29:27.417 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_98423fadeecbba811088c3136f8295dae6dfe870/qa/tasks 2024-04-12T21:29:27.418 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-12T21:29:27.419 INFO:teuthology.task.internal:Checking packages... 2024-04-12T21:29:27.443 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a' 2024-04-12T21:29:27.443 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T21:29:27.443 INFO:teuthology.packaging:ref: None 2024-04-12T21:29:27.443 INFO:teuthology.packaging:tag: None 2024-04-12T21:29:27.443 INFO:teuthology.packaging:branch: squid 2024-04-12T21:29:27.443 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T21:29:27.443 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-12T21:29:27.655 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2053.gb277feb2 2024-04-12T21:29:27.658 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-12T21:29:27.665 INFO:teuthology.task.internal:no buildpackages task found 2024-04-12T21:29:27.665 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-12T21:29:27.734 INFO:teuthology.task.internal:Saving configuration 2024-04-12T21:29:27.748 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-12T21:29:27.754 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T21:29:27.780 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi123.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 21:24:37.645509', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGmn6Phy9Qyz8bcmSfXEWGoGASyjs+Z70dhNTVJ+VNRDnDca15Dme2F1oo44XBwBm3Bws9Latr+JelBnYJF8Mfk='} 2024-04-12T21:29:27.804 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi177.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 21:24:37.646798', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP5navxQ0Zgey4yHcCk2usMWx4nnxrcWaXPPRSetGO/ESja/FSz912f94o5bW8g5Nze5LHJnfSHoODgUvdLqR/w='} 2024-04-12T21:29:27.805 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-12T21:29:27.811 INFO:teuthology.task.internal:roles: ubuntu@smithi123.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-12T21:29:27.811 INFO:teuthology.task.internal:roles: ubuntu@smithi177.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-12T21:29:27.811 INFO:teuthology.run_tasks:Running task console_log... 2024-04-12T21:29:27.887 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f0ef6dc4f70>, signals=[15]) 2024-04-12T21:29:27.888 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-12T21:29:27.894 INFO:teuthology.task.internal:Opening connections... 2024-04-12T21:29:27.894 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:29:27.896 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T21:29:27.989 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi177.front.sepia.ceph.com 2024-04-12T21:29:27.990 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T21:29:28.078 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-12T21:29:28.085 DEBUG:teuthology.orchestra.run.smithi123:> uname -m 2024-04-12T21:29:28.102 INFO:teuthology.orchestra.run.smithi123.stdout:x86_64 2024-04-12T21:29:28.103 DEBUG:teuthology.orchestra.run.smithi123:> cat /etc/os-release 2024-04-12T21:29:28.156 INFO:teuthology.orchestra.run.smithi123.stdout:NAME="CentOS Stream" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:VERSION="9" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:ID="centos" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:ID_LIKE="rhel fedora" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:VERSION_ID="9" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:PLATFORM_ID="platform:el9" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:ANSI_COLOR="0;31" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:LOGO="fedora-logo-icon" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:HOME_URL="https://centos.org/" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-12T21:29:28.157 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-12T21:29:28.158 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-12T21:29:28.158 INFO:teuthology.lock.ops:Updating smithi123.front.sepia.ceph.com on lock server 2024-04-12T21:29:28.176 DEBUG:teuthology.orchestra.run.smithi177:> uname -m 2024-04-12T21:29:28.194 INFO:teuthology.orchestra.run.smithi177.stdout:x86_64 2024-04-12T21:29:28.195 DEBUG:teuthology.orchestra.run.smithi177:> cat /etc/os-release 2024-04-12T21:29:28.255 INFO:teuthology.orchestra.run.smithi177.stdout:NAME="CentOS Stream" 2024-04-12T21:29:28.255 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION="9" 2024-04-12T21:29:28.255 INFO:teuthology.orchestra.run.smithi177.stdout:ID="centos" 2024-04-12T21:29:28.255 INFO:teuthology.orchestra.run.smithi177.stdout:ID_LIKE="rhel fedora" 2024-04-12T21:29:28.255 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION_ID="9" 2024-04-12T21:29:28.255 INFO:teuthology.orchestra.run.smithi177.stdout:PLATFORM_ID="platform:el9" 2024-04-12T21:29:28.255 INFO:teuthology.orchestra.run.smithi177.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-12T21:29:28.255 INFO:teuthology.orchestra.run.smithi177.stdout:ANSI_COLOR="0;31" 2024-04-12T21:29:28.256 INFO:teuthology.orchestra.run.smithi177.stdout:LOGO="fedora-logo-icon" 2024-04-12T21:29:28.256 INFO:teuthology.orchestra.run.smithi177.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-12T21:29:28.256 INFO:teuthology.orchestra.run.smithi177.stdout:HOME_URL="https://centos.org/" 2024-04-12T21:29:28.256 INFO:teuthology.orchestra.run.smithi177.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-12T21:29:28.256 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-12T21:29:28.256 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-12T21:29:28.256 INFO:teuthology.lock.ops:Updating smithi177.front.sepia.ceph.com on lock server 2024-04-12T21:29:28.277 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-12T21:29:28.286 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-12T21:29:28.291 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-12T21:29:28.292 DEBUG:teuthology.orchestra.run.smithi123:> test '!' -e /home/ubuntu/cephtest 2024-04-12T21:29:28.294 DEBUG:teuthology.orchestra.run.smithi177:> test '!' -e /home/ubuntu/cephtest 2024-04-12T21:29:28.315 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-12T21:29:28.322 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-12T21:29:28.322 DEBUG:teuthology.orchestra.run.smithi123:> test -z $(ls -A /var/lib/ceph) 2024-04-12T21:29:28.349 DEBUG:teuthology.orchestra.run.smithi177:> test -z $(ls -A /var/lib/ceph) 2024-04-12T21:29:28.441 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-12T21:29:28.499 INFO:teuthology.run_tasks:Running task kernel... 2024-04-12T21:29:28.513 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-12T21:29:28.513 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-12T21:29:28.514 DEBUG:teuthology.orchestra.run.smithi123:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-12T21:29:28.514 DEBUG:teuthology.orchestra.run.smithi177:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-12T21:29:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T21:29:28.530 DEBUG:teuthology.orchestra.run.smithi177:> uname -r 2024-04-12T21:29:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T21:29:28.531 DEBUG:teuthology.orchestra.run.smithi123:> uname -r 2024-04-12T21:29:28.587 INFO:teuthology.orchestra.run.smithi123.stdout:5.14.0-361.el9.x86_64 2024-04-12T21:29:28.587 INFO:teuthology.task.kernel:Running kernel on smithi123: 5.14.0-361.el9.x86_64 2024-04-12T21:29:28.587 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum install -y kernel 2024-04-12T21:29:28.589 INFO:teuthology.orchestra.run.smithi177.stdout:5.14.0-361.el9.x86_64 2024-04-12T21:29:28.589 INFO:teuthology.task.kernel:Running kernel on smithi177: 5.14.0-361.el9.x86_64 2024-04-12T21:29:28.590 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum install -y kernel 2024-04-12T21:29:30.804 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS-9 - AppStream 15 MB/s | 8.1 MB 00:00 2024-04-12T21:29:30.834 INFO:teuthology.orchestra.run.smithi177.stdout:CentOS-9 - AppStream 14 MB/s | 8.1 MB 00:00 2024-04-12T21:29:32.535 INFO:teuthology.orchestra.run.smithi177.stdout:CentOS-9 - BaseOS 6.7 MB/s | 2.5 MB 00:00 2024-04-12T21:29:32.612 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS-9 - BaseOS 5.2 MB/s | 2.5 MB 00:00 2024-04-12T21:29:33.467 INFO:teuthology.orchestra.run.smithi177.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.0 MB 00:00 2024-04-12T21:29:33.544 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.0 MB 00:00 2024-04-12T21:29:38.637 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - AppStream 4.7 MB/s | 19 MB 00:03 2024-04-12T21:29:41.117 INFO:teuthology.orchestra.run.smithi177.stdout:CentOS Stream 9 - AppStream 2.9 MB/s | 19 MB 00:06 2024-04-12T21:29:42.646 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.2 MB 00:00 2024-04-12T21:29:44.306 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - Extras packages 32 kB/s | 16 kB 00:00 2024-04-12T21:29:45.340 INFO:teuthology.orchestra.run.smithi123.stdout:Extra Packages for Enterprise Linux 24 MB/s | 21 MB 00:00 2024-04-12T21:29:45.393 INFO:teuthology.orchestra.run.smithi177.stdout:CentOS Stream 9 - CRB 9.2 MB/s | 6.2 MB 00:00 2024-04-12T21:29:47.743 INFO:teuthology.orchestra.run.smithi177.stdout:CentOS Stream 9 - Extras packages 14 kB/s | 16 kB 00:01 2024-04-12T21:29:49.411 INFO:teuthology.orchestra.run.smithi177.stdout:Extra Packages for Enterprise Linux 14 MB/s | 21 MB 00:01 2024-04-12T21:29:50.339 INFO:teuthology.orchestra.run.smithi123.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-04-12T21:29:52.572 INFO:teuthology.orchestra.run.smithi123.stderr:Modular dependency problems: 2024-04-12T21:29:52.572 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:29:52.572 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 1: conflicting requests 2024-04-12T21:29:52.572 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.572 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 2: conflicting requests 2024-04-12T21:29:52.572 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.572 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 3: conflicting requests 2024-04-12T21:29:52.572 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 4: conflicting requests 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 5: conflicting requests 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 6: conflicting requests 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 7: conflicting requests 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 8: conflicting requests 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 9: conflicting requests 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 10: conflicting requests 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.573 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 11: conflicting requests 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 12: conflicting requests 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 13: conflicting requests 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 14: conflicting requests 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 15: conflicting requests 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 16: conflicting requests 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 17: conflicting requests 2024-04-12T21:29:52.574 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 18: conflicting requests 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 19: conflicting requests 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 20: conflicting requests 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 21: conflicting requests 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 22: conflicting requests 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 23: conflicting requests 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 24: conflicting requests 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.575 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 25: conflicting requests 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 26: conflicting requests 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 27: conflicting requests 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 28: conflicting requests 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 29: conflicting requests 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 30: conflicting requests 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 31: conflicting requests 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 32: conflicting requests 2024-04-12T21:29:52.576 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 33: conflicting requests 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 34: conflicting requests 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 35: conflicting requests 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 36: conflicting requests 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 37: conflicting requests 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.577 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 39: conflicting requests 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.578 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.579 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.580 INFO:teuthology.orchestra.run.smithi123.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T21:29:52.971 INFO:teuthology.orchestra.run.smithi123.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-12T21:29:52.971 INFO:teuthology.orchestra.run.smithi123.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-12T21:29:53.074 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2024-04-12T21:29:53.082 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-04-12T21:29:53.082 INFO:teuthology.orchestra.run.smithi123.stdout: Package Architecture Version Repository Size 2024-04-12T21:29:53.082 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout:Install 4 Packages 2024-04-12T21:29:53.083 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:29:53.084 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 77 M 2024-04-12T21:29:53.084 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 123 M 2024-04-12T21:29:53.084 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2024-04-12T21:29:53.465 INFO:teuthology.orchestra.run.smithi123.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 863 kB/s | 214 kB 00:00 2024-04-12T21:29:53.833 INFO:teuthology.orchestra.run.smithi123.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 25 MB/s | 15 MB 00:00 2024-04-12T21:29:54.428 INFO:teuthology.orchestra.run.smithi177.stdout:lab-extras 35 kB/s | 1.7 kB 00:00 2024-04-12T21:29:54.574 INFO:teuthology.orchestra.run.smithi123.stdout:(3/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 25 MB/s | 34 MB 00:01 2024-04-12T21:29:54.833 INFO:teuthology.orchestra.run.smithi123.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 20 MB/s | 28 MB 00:01 2024-04-12T21:29:54.836 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2024-04-12T21:29:54.836 INFO:teuthology.orchestra.run.smithi123.stdout:Total 44 MB/s | 77 MB 00:01 2024-04-12T21:29:55.508 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2024-04-12T21:29:55.849 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2024-04-12T21:29:55.849 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2024-04-12T21:29:56.667 INFO:teuthology.orchestra.run.smithi177.stderr:Modular dependency problems: 2024-04-12T21:29:56.667 INFO:teuthology.orchestra.run.smithi177.stderr: 2024-04-12T21:29:56.667 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 1: conflicting requests 2024-04-12T21:29:56.667 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.667 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 2: conflicting requests 2024-04-12T21:29:56.667 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.667 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 3: conflicting requests 2024-04-12T21:29:56.667 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 4: conflicting requests 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 5: conflicting requests 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 6: conflicting requests 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 7: conflicting requests 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 8: conflicting requests 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 9: conflicting requests 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 10: conflicting requests 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 11: conflicting requests 2024-04-12T21:29:56.668 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 12: conflicting requests 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 13: conflicting requests 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 14: conflicting requests 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 15: conflicting requests 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 16: conflicting requests 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 17: conflicting requests 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 18: conflicting requests 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.669 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 19: conflicting requests 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 20: conflicting requests 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 21: conflicting requests 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 22: conflicting requests 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 23: conflicting requests 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 24: conflicting requests 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 25: conflicting requests 2024-04-12T21:29:56.670 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 26: conflicting requests 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 27: conflicting requests 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 28: conflicting requests 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 29: conflicting requests 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 30: conflicting requests 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 31: conflicting requests 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 32: conflicting requests 2024-04-12T21:29:56.671 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 33: conflicting requests 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 34: conflicting requests 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 35: conflicting requests 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 36: conflicting requests 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 37: conflicting requests 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.672 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 39: conflicting requests 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.673 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.674 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.675 INFO:teuthology.orchestra.run.smithi177.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T21:29:56.854 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2024-04-12T21:29:56.855 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2024-04-12T21:29:57.060 INFO:teuthology.orchestra.run.smithi177.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-12T21:29:57.060 INFO:teuthology.orchestra.run.smithi177.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-12T21:29:57.174 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2024-04-12T21:29:57.185 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout: Package Architecture Version Repository Size 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout:Installing: 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout:Installing dependencies: 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:29:57.186 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction Summary 2024-04-12T21:29:57.187 INFO:teuthology.orchestra.run.smithi177.stdout:================================================================================ 2024-04-12T21:29:57.187 INFO:teuthology.orchestra.run.smithi177.stdout:Install 4 Packages 2024-04-12T21:29:57.187 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:29:57.187 INFO:teuthology.orchestra.run.smithi177.stdout:Total download size: 77 M 2024-04-12T21:29:57.187 INFO:teuthology.orchestra.run.smithi177.stdout:Installed size: 123 M 2024-04-12T21:29:57.187 INFO:teuthology.orchestra.run.smithi177.stdout:Downloading Packages: 2024-04-12T21:29:57.696 INFO:teuthology.orchestra.run.smithi177.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 1.0 MB/s | 214 kB 00:00 2024-04-12T21:29:58.271 INFO:teuthology.orchestra.run.smithi177.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 19 MB/s | 15 MB 00:00 2024-04-12T21:29:58.863 INFO:teuthology.orchestra.run.smithi177.stdout:(3/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 25 MB/s | 34 MB 00:01 2024-04-12T21:29:59.213 INFO:teuthology.orchestra.run.smithi177.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 18 MB/s | 28 MB 00:01 2024-04-12T21:29:59.213 INFO:teuthology.orchestra.run.smithi177.stdout:-------------------------------------------------------------------------------- 2024-04-12T21:29:59.213 INFO:teuthology.orchestra.run.smithi177.stdout:Total 38 MB/s | 77 MB 00:02 2024-04-12T21:29:59.344 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2024-04-12T21:29:59.881 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction check 2024-04-12T21:29:59.926 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-12T21:30:00.016 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T21:30:00.221 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction check succeeded. 2024-04-12T21:30:00.222 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction test 2024-04-12T21:30:01.277 INFO:teuthology.orchestra.run.smithi177.stdout:Transaction test succeeded. 2024-04-12T21:30:01.277 INFO:teuthology.orchestra.run.smithi177.stdout:Running transaction 2024-04-12T21:30:02.773 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T21:30:02.902 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T21:30:03.799 INFO:teuthology.orchestra.run.smithi177.stdout: Preparing : 1/1 2024-04-12T21:30:04.399 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-12T21:30:04.539 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T21:30:07.316 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T21:30:07.458 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T21:30:07.614 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T21:30:07.695 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:12.190 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T21:30:12.234 INFO:teuthology.orchestra.run.smithi177.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:12.428 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:16.987 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:41.551 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:41.551 INFO:teuthology.orchestra.run.smithi123.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-12T21:30:41.551 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:30:41.596 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:41.936 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:41.936 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-12T21:30:41.936 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T21:30:41.936 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:30:42.357 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2024-04-12T21:30:42.704 DEBUG:teuthology.orchestra.run.smithi123:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T21:30:42.849 INFO:teuthology.orchestra.run.smithi123.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T21:30:42.849 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T21:30:42.850 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-12T21:30:42.850 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-12T21:30:42.850 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-12T21:30:42.850 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-04-12T21:30:42.850 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi123.front.sepia.ceph.com, path=None, version=distro) 2024-04-12T21:30:42.850 DEBUG:teuthology.orchestra.run.smithi123:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T21:30:42.931 INFO:teuthology.orchestra.run.smithi123.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T21:30:42.931 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T21:30:42.931 DEBUG:teuthology.orchestra.run.smithi123:> sudo rpm -qi grub2-tools 2024-04-12T21:30:42.975 INFO:teuthology.orchestra.run.smithi123.stdout:Name : grub2-tools 2024-04-12T21:30:42.975 INFO:teuthology.orchestra.run.smithi123.stdout:Epoch : 1 2024-04-12T21:30:42.975 INFO:teuthology.orchestra.run.smithi123.stdout:Version : 2.06 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Release : 68.el9 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Architecture: x86_64 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Group : Unspecified 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Size : 8264254 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:License : GPLv3+ 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Packager : builder@centos.org 2024-04-12T21:30:42.976 INFO:teuthology.orchestra.run.smithi123.stdout:Vendor : CentOS 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout:Summary : Support tools for GRUB. 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout:Description : 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout:hardware devices. 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:30:42.977 INFO:teuthology.orchestra.run.smithi123.stdout:This subpackage provides tools for support of all platforms. 2024-04-12T21:30:42.978 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-12T21:30:42.978 INFO:teuthology.task.kernel:Updating grub on smithi123 to boot 5.14.0-435.el9.x86_64 2024-04-12T21:30:42.978 DEBUG:teuthology.orchestra.run.smithi123:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-12T21:30:43.618 INFO:teuthology.orchestra.run.smithi123.stderr:Generating grub configuration file ... 2024-04-12T21:30:45.003 INFO:teuthology.orchestra.run.smithi123.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-12T21:30:45.032 INFO:teuthology.orchestra.run.smithi123.stderr:done 2024-04-12T21:30:45.035 DEBUG:teuthology.orchestra.run.smithi123:> mktemp 2024-04-12T21:30:45.049 INFO:teuthology.orchestra.run.smithi123.stdout:/tmp/tmp.HoCB4ZjiEM 2024-04-12T21:30:45.050 DEBUG:teuthology.orchestra.run.smithi123:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.HoCB4ZjiEM 2024-04-12T21:30:45.118 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 0666 /tmp/tmp.HoCB4ZjiEM 2024-04-12T21:30:45.256 DEBUG:teuthology.orchestra.remote:smithi123:/tmp/tmp.HoCB4ZjiEM is 6KB 2024-04-12T21:30:45.305 DEBUG:teuthology.orchestra.run.smithi123:> rm -fr /tmp/tmp.HoCB4ZjiEM 2024-04-12T21:30:45.319 DEBUG:teuthology.orchestra.run.smithi123:> sudo /bin/ls /boot/loader/entries || true 2024-04-12T21:30:45.390 INFO:teuthology.orchestra.run.smithi123.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-12T21:30:45.391 INFO:teuthology.orchestra.run.smithi123.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-12T21:30:45.391 INFO:teuthology.orchestra.run.smithi123.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-12T21:30:45.391 INFO:teuthology.orchestra.run.smithi123.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-12T21:30:45.392 DEBUG:teuthology.orchestra.run.smithi123:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-12T21:30:45.541 DEBUG:teuthology.orchestra.run.smithi123:> sudo shutdown -r now 2024-04-12T21:30:46.011 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:46.012 INFO:teuthology.orchestra.run.smithi177.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-12T21:30:46.012 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:30:46.056 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:46.461 INFO:teuthology.orchestra.run.smithi177.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:46.461 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-12T21:30:46.462 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T21:30:46.462 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout:Installed: 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:30:46.927 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2024-04-12T21:30:47.247 DEBUG:teuthology.orchestra.run.smithi177:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T21:30:47.353 INFO:teuthology.orchestra.run.smithi177.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T21:30:47.353 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T21:30:47.353 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-12T21:30:47.353 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-12T21:30:47.353 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-12T21:30:47.354 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-04-12T21:30:47.354 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi177.front.sepia.ceph.com, path=None, version=distro) 2024-04-12T21:30:47.354 DEBUG:teuthology.orchestra.run.smithi177:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T21:30:47.435 INFO:teuthology.orchestra.run.smithi177.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T21:30:47.435 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T21:30:47.436 DEBUG:teuthology.orchestra.run.smithi177:> sudo rpm -qi grub2-tools 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Name : grub2-tools 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Epoch : 1 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Version : 2.06 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Release : 68.el9 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Architecture: x86_64 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Group : Unspecified 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Size : 8264254 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:License : GPLv3+ 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-12T21:30:47.477 INFO:teuthology.orchestra.run.smithi177.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:Packager : builder@centos.org 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:Vendor : CentOS 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:Summary : Support tools for GRUB. 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:Description : 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:hardware devices. 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:30:47.478 INFO:teuthology.orchestra.run.smithi177.stdout:This subpackage provides tools for support of all platforms. 2024-04-12T21:30:47.479 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-12T21:30:47.479 INFO:teuthology.task.kernel:Updating grub on smithi177 to boot 5.14.0-435.el9.x86_64 2024-04-12T21:30:47.479 DEBUG:teuthology.orchestra.run.smithi177:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-12T21:30:48.109 INFO:teuthology.orchestra.run.smithi177.stderr:Generating grub configuration file ... 2024-04-12T21:30:49.505 INFO:teuthology.orchestra.run.smithi177.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-12T21:30:49.531 INFO:teuthology.orchestra.run.smithi177.stderr:done 2024-04-12T21:30:49.533 DEBUG:teuthology.orchestra.run.smithi177:> mktemp 2024-04-12T21:30:49.548 INFO:teuthology.orchestra.run.smithi177.stdout:/tmp/tmp.w5L4QM6VWH 2024-04-12T21:30:49.548 DEBUG:teuthology.orchestra.run.smithi177:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.w5L4QM6VWH 2024-04-12T21:30:49.621 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod 0666 /tmp/tmp.w5L4QM6VWH 2024-04-12T21:30:49.759 DEBUG:teuthology.orchestra.remote:smithi177:/tmp/tmp.w5L4QM6VWH is 6KB 2024-04-12T21:30:49.808 DEBUG:teuthology.orchestra.run.smithi177:> rm -fr /tmp/tmp.w5L4QM6VWH 2024-04-12T21:30:49.822 DEBUG:teuthology.orchestra.run.smithi177:> sudo /bin/ls /boot/loader/entries || true 2024-04-12T21:30:49.904 INFO:teuthology.orchestra.run.smithi177.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-12T21:30:49.904 INFO:teuthology.orchestra.run.smithi177.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-12T21:30:49.904 INFO:teuthology.orchestra.run.smithi177.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-12T21:30:49.904 INFO:teuthology.orchestra.run.smithi177.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-12T21:30:49.905 DEBUG:teuthology.orchestra.run.smithi177:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-12T21:30:50.041 DEBUG:teuthology.orchestra.run.smithi177:> sudo shutdown -r now 2024-04-12T21:31:15.557 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-04-12T21:31:15.558 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-04-12T21:31:15.559 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T21:31:20.048 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-04-12T21:31:20.048 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-04-12T21:31:20.049 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T21:32:15.613 DEBUG:teuthology.orchestra.remote:timed out 2024-04-12T21:32:20.054 DEBUG:teuthology.orchestra.remote:timed out 2024-04-12T21:32:24.620 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-04-12T21:32:24.621 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T21:32:24.937 DEBUG:teuthology.orchestra.run.smithi123:> true 2024-04-12T21:32:25.504 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-04-12T21:32:25.504 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-435.el9.x86_64"... 2024-04-12T21:32:25.504 DEBUG:teuthology.orchestra.run.smithi123:> uname -r 2024-04-12T21:32:25.519 INFO:teuthology.orchestra.run.smithi123.stdout:5.14.0-435.el9.x86_64 2024-04-12T21:32:25.519 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-12T21:32:25.519 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-12T21:32:25.519 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-12T21:32:26.520 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-12T21:32:26.520 DEBUG:teuthology.orchestra.run.smithi123:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-12T21:32:26.593 INFO:teuthology.orchestra.run.smithi123.stdout:ttyS1 2024-04-12T21:32:26.641 DEBUG:teuthology.parallel:result is None 2024-04-12T21:32:29.055 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-04-12T21:32:29.055 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T21:32:29.412 DEBUG:teuthology.orchestra.run.smithi177:> true 2024-04-12T21:32:29.818 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-04-12T21:32:29.818 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-435.el9.x86_64"... 2024-04-12T21:32:29.818 DEBUG:teuthology.orchestra.run.smithi177:> uname -r 2024-04-12T21:32:29.832 INFO:teuthology.orchestra.run.smithi177.stdout:5.14.0-435.el9.x86_64 2024-04-12T21:32:29.832 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-12T21:32:29.832 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-12T21:32:29.832 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-12T21:32:30.833 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-12T21:32:30.834 DEBUG:teuthology.orchestra.run.smithi177:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-12T21:32:30.901 INFO:teuthology.orchestra.run.smithi177.stdout:ttyS1 2024-04-12T21:32:30.944 DEBUG:teuthology.parallel:result is None 2024-04-12T21:32:30.944 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-12T21:32:30.951 INFO:teuthology.task.internal:Creating test directory... 2024-04-12T21:32:30.952 DEBUG:teuthology.orchestra.run.smithi123:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-12T21:32:30.954 DEBUG:teuthology.orchestra.run.smithi177:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-12T21:32:30.968 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-12T21:32:30.974 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-12T21:32:30.983 INFO:teuthology.task.internal:Creating archive directory... 2024-04-12T21:32:30.983 DEBUG:teuthology.orchestra.run.smithi123:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-12T21:32:31.006 DEBUG:teuthology.orchestra.run.smithi177:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-12T21:32:31.050 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-12T21:32:31.057 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-12T21:32:31.057 DEBUG:teuthology.orchestra.run.smithi123:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-12T21:32:31.087 DEBUG:teuthology.orchestra.run.smithi177:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-12T21:32:31.121 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T21:32:31.134 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T21:32:31.350 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T21:32:31.372 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T21:32:31.374 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-12T21:32:31.381 INFO:teuthology.task.internal:Configuring sudo... 2024-04-12T21:32:31.382 DEBUG:teuthology.orchestra.run.smithi123:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-12T21:32:31.384 DEBUG:teuthology.orchestra.run.smithi177:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-12T21:32:31.430 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-12T21:32:31.439 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-12T21:32:31.440 DEBUG:teuthology.orchestra.run.smithi123:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-12T21:32:31.442 DEBUG:teuthology.orchestra.run.smithi177:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-12T21:32:31.466 DEBUG:teuthology.orchestra.run.smithi123:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T21:32:31.552 DEBUG:teuthology.orchestra.run.smithi123:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T21:32:31.611 DEBUG:teuthology.orchestra.run.smithi123:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T21:32:31.664 DEBUG:teuthology.orchestra.run.smithi123:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T21:32:31.719 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:32:31.719 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-12T21:32:31.765 DEBUG:teuthology.orchestra.run.smithi177:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T21:32:31.806 DEBUG:teuthology.orchestra.run.smithi177:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T21:32:31.859 DEBUG:teuthology.orchestra.run.smithi177:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T21:32:31.941 DEBUG:teuthology.orchestra.run.smithi177:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T21:32:31.983 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-04-12T21:32:31.983 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-12T21:32:32.052 DEBUG:teuthology.orchestra.run.smithi123:> sudo service rsyslog restart 2024-04-12T21:32:32.054 DEBUG:teuthology.orchestra.run.smithi177:> sudo service rsyslog restart 2024-04-12T21:32:32.097 INFO:teuthology.orchestra.run.smithi123.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T21:32:32.143 INFO:teuthology.orchestra.run.smithi177.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T21:32:32.369 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-12T21:32:32.378 INFO:teuthology.task.internal:Starting timer... 2024-04-12T21:32:32.378 INFO:teuthology.run_tasks:Running task pcp... 2024-04-12T21:32:32.387 INFO:teuthology.run_tasks:Running task selinux... 2024-04-12T21:32:32.424 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-12T21:32:32.425 DEBUG:teuthology.orchestra.run.smithi123:> sudo service auditd rotate 2024-04-12T21:32:32.590 INFO:teuthology.orchestra.run.smithi123.stdout:Rotating logs: 2024-04-12T21:32:32.592 DEBUG:teuthology.orchestra.run.smithi177:> sudo service auditd rotate 2024-04-12T21:32:32.649 INFO:teuthology.orchestra.run.smithi177.stdout:Rotating logs: 2024-04-12T21:32:32.651 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-12T21:32:32.651 DEBUG:teuthology.orchestra.run.smithi123:> /usr/sbin/getenforce 2024-04-12T21:32:32.697 INFO:teuthology.orchestra.run.smithi123.stdout:Permissive 2024-04-12T21:32:32.697 DEBUG:teuthology.orchestra.run.smithi177:> /usr/sbin/getenforce 2024-04-12T21:32:32.719 INFO:teuthology.orchestra.run.smithi177.stdout:Permissive 2024-04-12T21:32:32.720 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi123.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi177.front.sepia.ceph.com': 'permissive'} 2024-04-12T21:32:32.720 DEBUG:teuthology.orchestra.run.smithi123:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-12T21:32:32.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T21:32:32.747 DEBUG:teuthology.orchestra.run.smithi177:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-12T21:32:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T21:32:32.791 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-12T21:32:32.792 DEBUG:teuthology.orchestra.run.smithi123:> sudo /usr/sbin/setenforce permissive 2024-04-12T21:32:32.866 DEBUG:teuthology.orchestra.run.smithi177:> sudo /usr/sbin/setenforce permissive 2024-04-12T21:32:32.905 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-12T21:32:32.915 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-12T21:32:32.921 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-12T21:32:33.049 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-12T21:32:33.070 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-12T21:32:33.072 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi123.front.sepia.ceph.com,smithi177.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-12T21:36:55.025 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi123.front.sepia.ceph.com'), Remote(name='ubuntu@smithi177.front.sepia.ceph.com')] 2024-04-12T21:36:55.027 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-04-12T21:36:55.028 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T21:36:55.120 DEBUG:teuthology.orchestra.run.smithi123:> true 2024-04-12T21:36:55.196 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-04-12T21:36:55.197 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-04-12T21:36:55.197 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T21:36:55.282 DEBUG:teuthology.orchestra.run.smithi177:> true 2024-04-12T21:36:55.364 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi177.front.sepia.ceph.com' 2024-04-12T21:36:55.364 INFO:teuthology.run_tasks:Running task clock... 2024-04-12T21:36:55.376 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-12T21:36:55.376 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-12T21:36:55.376 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T21:36:55.378 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-12T21:36:55.378 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T21:36:55.407 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-12T21:36:55.424 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-12T21:36:55.435 INFO:teuthology.orchestra.run.smithi177.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-12T21:36:55.451 INFO:teuthology.orchestra.run.smithi177.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-12T21:36:55.452 INFO:teuthology.orchestra.run.smithi123.stderr:sudo: ntpd: command not found 2024-04-12T21:36:55.466 INFO:teuthology.orchestra.run.smithi123.stdout:506 Cannot talk to daemon 2024-04-12T21:36:55.480 INFO:teuthology.orchestra.run.smithi177.stderr:sudo: ntpd: command not found 2024-04-12T21:36:55.482 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-12T21:36:55.493 INFO:teuthology.orchestra.run.smithi177.stdout:506 Cannot talk to daemon 2024-04-12T21:36:55.498 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-12T21:36:55.510 INFO:teuthology.orchestra.run.smithi177.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-12T21:36:55.526 INFO:teuthology.orchestra.run.smithi177.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-12T21:36:55.553 INFO:teuthology.orchestra.run.smithi123.stderr:bash: line 1: ntpq: command not found 2024-04-12T21:36:55.555 INFO:teuthology.orchestra.run.smithi123.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T21:36:55.556 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================== 2024-04-12T21:36:55.556 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T21:36:55.556 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T21:36:55.556 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T21:36:55.556 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T21:36:55.580 INFO:teuthology.orchestra.run.smithi177.stderr:bash: line 1: ntpq: command not found 2024-04-12T21:36:55.584 INFO:teuthology.orchestra.run.smithi177.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T21:36:55.584 INFO:teuthology.orchestra.run.smithi177.stdout:=============================================================================== 2024-04-12T21:36:55.584 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T21:36:55.584 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T21:36:55.585 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T21:36:55.585 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T21:36:55.585 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-12T21:36:55.671 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a'} 2024-04-12T21:36:55.672 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-12T21:36:55.672 INFO:tasks.cephadm:Cluster fsid is c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:36:55.672 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-12T21:36:55.672 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-12T21:36:55.672 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi123': '172.21.15.123', 'mon.smithi177': '172.21.15.177'} 2024-04-12T21:36:55.672 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-12T21:36:55.673 DEBUG:teuthology.orchestra.run.smithi123:> sudo hostname $(hostname -s) 2024-04-12T21:36:55.700 DEBUG:teuthology.orchestra.run.smithi177:> sudo hostname $(hostname -s) 2024-04-12T21:36:55.728 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-12T21:36:55.728 DEBUG:teuthology.orchestra.run.smithi123:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-12T21:36:56.009 INFO:teuthology.orchestra.run.smithi123.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 12 21:36 /home/ubuntu/cephtest/cephadm 2024-04-12T21:36:56.010 DEBUG:teuthology.orchestra.run.smithi177:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-12T21:36:56.171 INFO:teuthology.orchestra.run.smithi177.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 12 21:36 /home/ubuntu/cephtest/cephadm 2024-04-12T21:36:56.174 DEBUG:teuthology.orchestra.run.smithi123:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-12T21:36:56.189 DEBUG:teuthology.orchestra.run.smithi177:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-12T21:36:56.213 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-12T21:36:56.213 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-12T21:36:56.232 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-12T21:36:57.106 INFO:teuthology.orchestra.run.smithi123.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-12T21:36:57.128 INFO:teuthology.orchestra.run.smithi177.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-12T21:37:19.853 INFO:teuthology.orchestra.run.smithi177.stdout:{ 2024-04-12T21:37:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-12T21:37:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-12T21:37:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "repo_digests": [ 2024-04-12T21:37:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-12T21:37:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-12T21:37:19.854 INFO:teuthology.orchestra.run.smithi177.stdout: ] 2024-04-12T21:37:19.854 INFO:teuthology.orchestra.run.smithi177.stdout:} 2024-04-12T21:37:20.095 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:37:20.095 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-12T21:37:20.095 INFO:teuthology.orchestra.run.smithi123.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-12T21:37:20.095 INFO:teuthology.orchestra.run.smithi123.stdout: "repo_digests": [ 2024-04-12T21:37:20.095 INFO:teuthology.orchestra.run.smithi123.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-12T21:37:20.095 INFO:teuthology.orchestra.run.smithi123.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-12T21:37:20.096 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2024-04-12T21:37:20.096 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:37:20.114 DEBUG:teuthology.orchestra.run.smithi123:> sudo mkdir -p /etc/ceph 2024-04-12T21:37:20.156 DEBUG:teuthology.orchestra.run.smithi177:> sudo mkdir -p /etc/ceph 2024-04-12T21:37:20.187 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 777 /etc/ceph 2024-04-12T21:37:20.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo chmod 777 /etc/ceph 2024-04-12T21:37:20.255 INFO:tasks.cephadm:Writing seed config... 2024-04-12T21:37:20.256 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-12T21:37:20.256 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-12T21:37:20.256 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-12T21:37:20.256 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-12T21:37:20.256 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-12T21:37:20.256 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-12T21:37:20.256 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-12T21:37:20.256 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-12T21:37:20.257 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:37:20.257 DEBUG:teuthology.orchestra.run.smithi123:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-12T21:37:20.280 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = c8b0510a-f914-11ee-bc8d-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-12T21:37:20.280 DEBUG:teuthology.orchestra.run.smithi123:mon.smithi123> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123.service 2024-04-12T21:37:20.322 INFO:tasks.cephadm:Bootstrapping... 2024-04-12T21:37:20.323 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.123 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-12T21:37:20.523 INFO:teuthology.orchestra.run.smithi123.stderr:-------------------------------------------------------------------------------- 2024-04-12T21:37:20.523 INFO:teuthology.orchestra.run.smithi123.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'c8b0510a-f914-11ee-bc8d-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.123', '--skip-admin-label'] 2024-04-12T21:37:20.545 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: 4.9.4-dev 2024-04-12T21:37:20.548 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying podman|docker is present... 2024-04-12T21:37:20.568 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: 4.9.4-dev 2024-04-12T21:37:20.570 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying lvm2 is present... 2024-04-12T21:37:20.570 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying time synchronization is in place... 2024-04-12T21:37:20.577 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-12T21:37:20.583 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-04-12T21:37:20.590 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: enabled 2024-04-12T21:37:20.596 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: active 2024-04-12T21:37:20.596 INFO:teuthology.orchestra.run.smithi123.stderr:Unit chronyd.service is enabled and running 2024-04-12T21:37:20.596 INFO:teuthology.orchestra.run.smithi123.stderr:Repeating the final host check... 2024-04-12T21:37:20.617 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: 4.9.4-dev 2024-04-12T21:37:20.619 INFO:teuthology.orchestra.run.smithi123.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-12T21:37:20.619 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl is present 2024-04-12T21:37:20.619 INFO:teuthology.orchestra.run.smithi123.stderr:lvcreate is present 2024-04-12T21:37:20.625 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-12T21:37:20.631 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-04-12T21:37:20.637 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: enabled 2024-04-12T21:37:20.643 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: active 2024-04-12T21:37:20.644 INFO:teuthology.orchestra.run.smithi123.stderr:Unit chronyd.service is enabled and running 2024-04-12T21:37:20.644 INFO:teuthology.orchestra.run.smithi123.stderr:Host looks OK 2024-04-12T21:37:20.644 INFO:teuthology.orchestra.run.smithi123.stderr:Cluster fsid: c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:20.644 INFO:teuthology.orchestra.run.smithi123.stderr:Acquiring lock 139735574679904 on /run/cephadm/c8b0510a-f914-11ee-bc8d-c7b262605968.lock 2024-04-12T21:37:20.645 INFO:teuthology.orchestra.run.smithi123.stderr:Lock 139735574679904 acquired on /run/cephadm/c8b0510a-f914-11ee-bc8d-c7b262605968.lock 2024-04-12T21:37:20.645 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying IP 172.21.15.123 port 3300 ... 2024-04-12T21:37:20.645 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying IP 172.21.15.123 port 6789 ... 2024-04-12T21:37:20.646 INFO:teuthology.orchestra.run.smithi123.stderr:Base mon IP is 172.21.15.123, final addrv is [v2:172.21.15.123:3300,v1:172.21.15.123:6789] 2024-04-12T21:37:20.648 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.123 metric 100 2024-04-12T21:37:20.648 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.123 metric 100 2024-04-12T21:37:20.650 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-12T21:37:20.651 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-12T21:37:20.651 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1790sec hoplimit 64 pref medium 2024-04-12T21:37:20.653 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-12T21:37:20.653 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-12T21:37:20.653 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-12T21:37:20.653 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-12T21:37:20.653 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d3f1/64 scope link 2024-04-12T21:37:20.653 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-12T21:37:20.654 INFO:teuthology.orchestra.run.smithi123.stderr:Mon IP `172.21.15.123` is in CIDR network `172.21.0.0/20` 2024-04-12T21:37:20.654 INFO:teuthology.orchestra.run.smithi123.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-12T21:37:20.655 INFO:teuthology.orchestra.run.smithi123.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-12T21:37:20.755 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-12T21:37:21.137 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Getting image source signatures 2024-04-12T21:37:21.137 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-12T21:37:21.137 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-12T21:37:21.137 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-12T21:37:21.137 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-12T21:37:21.137 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-12T21:37:21.137 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-12T21:37:21.274 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Writing manifest to image destination 2024-04-12T21:37:21.376 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-12T21:37:21.776 INFO:teuthology.orchestra.run.smithi123.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-12T21:37:22.333 INFO:teuthology.orchestra.run.smithi123.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-12T21:37:22.333 INFO:teuthology.orchestra.run.smithi123.stderr:Extracting ceph user uid/gid from container image... 2024-04-12T21:37:22.716 INFO:teuthology.orchestra.run.smithi123.stderr:stat: 167 167 2024-04-12T21:37:23.164 INFO:teuthology.orchestra.run.smithi123.stderr:Creating initial keys... 2024-04-12T21:37:23.580 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph-authtool: AQCTqRlmvmeUHhAAf+yXdVbFJUodtx6+cRMjhQ== 2024-04-12T21:37:24.427 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph-authtool: AQCUqRlmPRbqFRAArRpcaybehVc5DnVUnNRpag== 2024-04-12T21:37:25.315 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph-authtool: AQCVqRlmRxDPDBAAuTTpeifxhRP081uL0FRy0w== 2024-04-12T21:37:25.972 INFO:teuthology.orchestra.run.smithi123.stderr:Creating initial monmap... 2024-04-12T21:37:26.363 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-12T21:37:26.363 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-12T21:37:26.363 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:26.363 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-12T21:37:26.868 INFO:teuthology.orchestra.run.smithi123.stderr:monmaptool for smithi123 [v2:172.21.15.123:3300,v1:172.21.15.123:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-12T21:37:26.868 INFO:teuthology.orchestra.run.smithi123.stderr:setting min_mon_release = octopus 2024-04-12T21:37:26.868 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: set fsid to c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:26.868 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-12T21:37:26.868 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:37:26.868 INFO:teuthology.orchestra.run.smithi123.stderr:Creating mon... 2024-04-12T21:37:27.709 INFO:teuthology.orchestra.run.smithi123.stderr:create mon.smithi123 on 2024-04-12T21:37:27.921 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-12T21:37:28.062 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968.target → /etc/systemd/system/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968.target. 2024-04-12T21:37:28.062 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968.target → /etc/systemd/system/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968.target. 2024-04-12T21:37:28.350 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Failed to reset failed state of unit ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123.service: Unit ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123.service not loaded. 2024-04-12T21:37:28.357 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968.target.wants/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123.service → /etc/systemd/system/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@.service. 2024-04-12T21:37:29.009 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:28 smithi123 podman[24476]: 2024-04-12 21:37:28.970216394 +0000 UTC m=+0.339739702 container start fc81313b120b1d76d53112442f5ced458c8770613a597bc3e969e3bbdcc38efc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.expose-services=, ceph=True, release=754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.buildah.version=1.19.8, vcs-type=git, version=8, build-date=2022-05-03T08:36:31.336870) 2024-04-12T21:37:29.009 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:29 smithi123 ceph-mon[24495]: mkfs c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:29.050 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: disabled 2024-04-12T21:37:29.057 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-04-12T21:37:29.058 INFO:teuthology.orchestra.run.smithi123.stderr:firewalld.service is not enabled 2024-04-12T21:37:29.058 INFO:teuthology.orchestra.run.smithi123.stderr:Not possible to enable service . firewalld.service is not available 2024-04-12T21:37:29.059 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mon to start... 2024-04-12T21:37:29.059 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mon... 2024-04-12T21:37:29.265 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:29 smithi123 ceph-mon[24495]: mon.smithi123 is new leader, mons smithi123 in quorum (ranks 0) 2024-04-12T21:37:29.265 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:29 smithi123 bash[24476]: fc81313b120b1d76d53112442f5ced458c8770613a597bc3e969e3bbdcc38efc 2024-04-12T21:37:29.265 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:29 smithi123 systemd[1]: Started Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:37:29.522 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: cluster: 2024-04-12T21:37:29.522 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: id: c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:29.522 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: services: 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi123 (age 0.514837s) 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: data: 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: pgs: 2024-04-12T21:37:29.523 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:30.108 INFO:teuthology.orchestra.run.smithi123.stderr:mon is available 2024-04-12T21:37:30.108 INFO:teuthology.orchestra.run.smithi123.stderr:Assimilating anything we can from ceph.conf... 2024-04-12T21:37:30.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:30 smithi123 ceph-mon[24495]: mon.smithi123 is new leader, mons smithi123 in quorum (ranks 0) 2024-04-12T21:37:30.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:30 smithi123 ceph-mon[24495]: monmap e1: 1 mons at {smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0]} 2024-04-12T21:37:30.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:30 smithi123 ceph-mon[24495]: fsmap 2024-04-12T21:37:30.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:30 smithi123 ceph-mon[24495]: osdmap e1: 0 total, 0 up, 0 in 2024-04-12T21:37:30.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:30 smithi123 ceph-mon[24495]: mgrmap e1: no daemons active 2024-04-12T21:37:30.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:30 smithi123 ceph-mon[24495]: from='client.? 172.21.15.123:0/312874518' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-12T21:37:30.607 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [global] 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: fsid = c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.123:3300,v1:172.21.15.123:6789] 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [mgr] 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [osd] 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-12T21:37:30.608 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-12T21:37:31.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:31 smithi123 ceph-mon[24495]: from='client.? 172.21.15.123:0/3707923378' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-12T21:37:31.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:31 smithi123 ceph-mon[24495]: from='client.? 172.21.15.123:0/3707923378' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-12T21:37:36.168 INFO:teuthology.orchestra.run.smithi123.stderr:Generating new minimal ceph.conf... 2024-04-12T21:37:40.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:40 smithi123 ceph-mon[24495]: from='client.? 172.21.15.123:0/983462913' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:37:43.125 INFO:teuthology.orchestra.run.smithi123.stderr:Restarting the monitor... 2024-04-12T21:37:43.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 systemd[1]: Stopping Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:37:43.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 bash[24715]: Error: no container with name or ID "ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon.smithi123" found: no such container 2024-04-12T21:37:43.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[24491]: 2024-04-12T21:37:43.258+0000 7ff3a34f1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi123 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:37:43.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[24491]: 2024-04-12T21:37:43.258+0000 7ff3a34f1700 -1 mon.smithi123@0(leader) e1 *** Got Signal Terminated *** 2024-04-12T21:37:43.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 podman[24726]: 2024-04-12 21:37:43.279813036 +0000 UTC m=+0.098155080 container died fc81313b120b1d76d53112442f5ced458c8770613a597bc3e969e3bbdcc38efc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, release=754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_CLEAN=True) 2024-04-12T21:37:43.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 podman[24726]: 2024-04-12 21:37:43.633789155 +0000 UTC m=+0.452131203 container cleanup fc81313b120b1d76d53112442f5ced458c8770613a597bc3e969e3bbdcc38efc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, io.buildah.version=1.19.8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, name=centos-stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container) 2024-04-12T21:37:43.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 bash[24726]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123 2024-04-12T21:37:43.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 bash[24753]: Error: no container with name or ID "ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon.smithi123" found: no such container 2024-04-12T21:37:44.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 podman[24740]: 2024-04-12 21:37:43.876317956 +0000 UTC m=+0.591443463 container remove fc81313b120b1d76d53112442f5ced458c8770613a597bc3e969e3bbdcc38efc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, GIT_CLEAN=True, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, distribution-scope=public, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-12T21:37:44.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123.service: Deactivated successfully. 2024-04-12T21:37:44.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 systemd[1]: Stopped Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:37:44.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:43 smithi123 systemd[1]: Starting Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:37:44.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 podman[24824]: 2024-04-12 21:37:44.011298981 +0000 UTC m=+0.018251887 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-12T21:37:44.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 podman[24824]: 2024-04-12 21:37:44.11024037 +0000 UTC m=+0.117193275 container create 8e8322580819e87f2a5f1b3e8b359afbce14d7277f3f1011ff6143a8d262a6b1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, distribution-scope=public, vendor=Red Hat, Inc., version=8, architecture=x86_64, io.buildah.version=1.19.8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.tags=base centos centos-stream) 2024-04-12T21:37:44.364 INFO:teuthology.orchestra.run.smithi123.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-12T21:37:44.471 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 podman[24824]: 2024-04-12 21:37:44.287477483 +0000 UTC m=+0.294430389 container init 8e8322580819e87f2a5f1b3e8b359afbce14d7277f3f1011ff6143a8d262a6b1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, release=754, vendor=Red Hat, Inc., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, ceph=True, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_BRANCH=HEAD) 2024-04-12T21:37:44.471 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 podman[24824]: 2024-04-12 21:37:44.29041669 +0000 UTC m=+0.297369596 container start 8e8322580819e87f2a5f1b3e8b359afbce14d7277f3f1011ff6143a8d262a6b1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.buildah.version=1.19.8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, version=8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-12T21:37:44.471 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T21:37:44.471 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-12T21:37:44.471 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: pidfile_write: ignore empty --pid-file 2024-04-12T21:37:44.471 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: load: jerasure load: lrc 2024-04-12T21:37:44.471 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: RocksDB version: 6.15.5 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Compile date Apr 18 2022 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: DB SUMMARY 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: DB Session ID: PRJDMMJFWRHB0V20IKYG 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: CURRENT file: CURRENT 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi123/store.db dir, Total Num: 1, files: 000008.sst 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi123/store.db: 000010.log size: 83737 ; 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.error_if_exists: 0 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.create_if_missing: 0 2024-04-12T21:37:44.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.paranoid_checks: 1 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.env: 0x556b8b6c6860 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.fs: Posix File System 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.info_log: 0x556b8c959fa0 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.statistics: (nil) 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.use_fsync: 0 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_log_file_size: 0 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.allow_fallocate: 1 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T21:37:44.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.use_direct_reads: 0 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.db_log_dir: 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi123/store.db 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.write_buffer_manager: 0x556b8ca4a2d0 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-12T21:37:44.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.unordered_write: 0 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.row_cache: None 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.wal_filter: None 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T21:37:44.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.preserve_deletes: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.two_write_queues: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.atomic_flush: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.log_readahead_size: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T21:37:44.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_background_jobs: 2 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_background_compactions: -1 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_subcompactions: 1 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_open_files: -1 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T21:37:44.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_background_flushes: -1 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Compression algorithms supported: 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: kZSTD supported: 0 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: kXpressCompression supported: 0 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: kLZ4Compression supported: 1 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: kBZip2Compression supported: 0 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: kZlibCompression supported: 1 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: kSnappyCompression supported: 1 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi123/store.db/MANIFEST-000009 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T21:37:44.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.merge_operator: 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_filter: None 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556b8c925d00) 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: cache_index_and_filter_blocks: 1 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: pin_top_level_index_and_filter: 1 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_type: 0 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_block_index_type: 0 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_shortening: 1 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T21:37:44.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: hash_index_allow_collision: 1 2024-04-12T21:37:44.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: checksum: 1 2024-04-12T21:37:44.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: no_block_cache: 0 2024-04-12T21:37:44.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache: 0x556b8c990170 2024-04-12T21:37:44.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_name: BinnedLRUCache 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_options: 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: capacity : 536870912 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: num_shard_bits : 4 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: strict_capacity_limit : 0 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: high_pri_pool_ratio: 0.000 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_compressed: (nil) 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: persistent_cache: (nil) 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_size: 4096 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_size_deviation: 10 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_restart_interval: 16 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_block_restart_interval: 1 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: metadata_block_size: 4096 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: partition_filters: 0 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: use_delta_encoding: 1 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-12T21:37:44.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: whole_key_filtering: 1 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: verify_compression: 0 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: read_amp_bytes_per_bit: 0 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: format_version: 4 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: enable_index_compression: 1 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_align: 0 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compression: NoCompression 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.num_levels: 7 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T21:37:44.482 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T21:37:44.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T21:37:44.484 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.arena_block_size: 4194304 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.table_properties_collectors: 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.inplace_update_support: 0 2024-04-12T21:37:44.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.bloom_locality: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.max_successive_merges: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.ttl: 2592000 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.enable_blob_files: false 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.min_blob_size: 0 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T21:37:44.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi123/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712957864315284, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712957864316046, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79967, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78214, "index_size": 241, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9916, "raw_average_key_size": 46, "raw_value_size": 72478, "raw_average_value_size": 343, "num_data_blocks": 11, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712957864, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f961ae73-0892-4bc7-b66d-c52f9795baea", "db_session_id": "PRJDMMJFWRHB0V20IKYG"}} 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712957864316365, "job": 1, "event": "recovery_finished"} 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi123/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T21:37:44.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x556b8c976a80 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: DB pointer 0x556b8c986000 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** DB Stats ** 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:37:44.488 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-04-12T21:37:44.489 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T21:37:44.489 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:37:44.489 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: L0 2/0 79.89 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.2 0.00 0.00 1 0.001 0 0 2024-04-12T21:37:44.489 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Sum 2/0 79.89 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.2 0.00 0.00 1 0.001 0 0 2024-04-12T21:37:44.489 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.2 0.00 0.00 1 0.001 0 0 2024-04-12T21:37:44.489 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 144.2 0.00 0.00 1 0.001 0 0 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative compaction: 0.00 GB write, 26.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval compaction: 0.00 GB write, 26.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:37:44.490 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: L0 2/0 79.89 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.2 0.00 0.00 1 0.001 0 0 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Sum 2/0 79.89 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.2 0.00 0.00 1 0.001 0 0 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 144.2 0.00 0.00 1 0.001 0 0 2024-04-12T21:37:44.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative compaction: 0.00 GB write, 26.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T21:37:44.492 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: starting mon.smithi123 rank 0 at public addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] at bind addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi123 fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123@-1(???) e1 preinit fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123@-1(???).mds e1 new map 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123@-1(???).mds e1 print_map 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: e1 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: legacy client fscid: -1 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: No filesystems configured 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T21:37:44.493 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T21:37:44.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-12T21:37:44.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 bash[24824]: 8e8322580819e87f2a5f1b3e8b359afbce14d7277f3f1011ff6143a8d262a6b1 2024-04-12T21:37:44.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 systemd[1]: Started Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:37:44.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mon.smithi123 is new leader, mons smithi123 in quorum (ranks 0) 2024-04-12T21:37:44.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: monmap e1: 1 mons at {smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0]} 2024-04-12T21:37:44.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: fsmap 2024-04-12T21:37:44.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: osdmap e1: 0 total, 0 up, 0 in 2024-04-12T21:37:44.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:44 smithi123 ceph-mon[24842]: mgrmap e1: no daemons active 2024-04-12T21:37:45.381 INFO:teuthology.orchestra.run.smithi123.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-12T21:37:45.381 INFO:teuthology.orchestra.run.smithi123.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-12T21:37:45.381 INFO:teuthology.orchestra.run.smithi123.stderr:Creating mgr... 2024-04-12T21:37:45.381 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying port 9283 ... 2024-04-12T21:37:45.549 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Failed to reset failed state of unit ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mgr.smithi123.psjrfs.service: Unit ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mgr.smithi123.psjrfs.service not loaded. 2024-04-12T21:37:45.556 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968.target.wants/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mgr.smithi123.psjrfs.service → /etc/systemd/system/ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@.service. 2024-04-12T21:37:46.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:45 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/565667218' entity='client.admin' 2024-04-12T21:37:46.213 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: disabled 2024-04-12T21:37:46.220 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-04-12T21:37:46.221 INFO:teuthology.orchestra.run.smithi123.stderr:firewalld.service is not enabled 2024-04-12T21:37:46.221 INFO:teuthology.orchestra.run.smithi123.stderr:Not possible to enable service . firewalld.service is not available 2024-04-12T21:37:46.229 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: disabled 2024-04-12T21:37:46.236 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-04-12T21:37:46.236 INFO:teuthology.orchestra.run.smithi123.stderr:firewalld.service is not enabled 2024-04-12T21:37:46.237 INFO:teuthology.orchestra.run.smithi123.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-12T21:37:46.237 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mgr to start... 2024-04-12T21:37:46.237 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mgr... 2024-04-12T21:37:46.696 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsid": "c8b0510a-f914-11ee-bc8d-c7b262605968", 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "health": { 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "checks": {}, 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mutes": [] 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum": [ 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 0 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "smithi123" 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:46.697 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "monmap": { 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osdmap": { 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-12T21:37:46.698 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-12T21:37:46.701 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:46.701 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgmap": { 2024-04-12T21:37:46.701 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsmap": { 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-12T21:37:46.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": false, 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modules": [ 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "iostat", 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "nfs", 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "restful" 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "servicemap": { 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modified": "2024-04-12T21:37:29.008157+0000", 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-04-12T21:37:46.703 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:46.704 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-12T21:37:46.704 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-04-12T21:37:47.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:46 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/174963931' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T21:37:47.214 INFO:teuthology.orchestra.run.smithi123.stderr:mgr not available, waiting (1/15)... 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsid": "c8b0510a-f914-11ee-bc8d-c7b262605968", 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "health": { 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "checks": {}, 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mutes": [] 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum": [ 2024-04-12T21:37:49.783 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 0 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "smithi123" 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "monmap": { 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osdmap": { 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-12T21:37:49.784 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-12T21:37:49.785 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-12T21:37:49.788 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-12T21:37:49.788 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-12T21:37:49.788 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:49.788 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgmap": { 2024-04-12T21:37:49.788 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-12T21:37:49.788 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-12T21:37:49.788 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-12T21:37:49.788 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsmap": { 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": false, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modules": [ 2024-04-12T21:37:49.789 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "iostat", 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "nfs", 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "restful" 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "servicemap": { 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modified": "2024-04-12T21:37:29.008157+0000", 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-12T21:37:49.790 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-04-12T21:37:50.317 INFO:teuthology.orchestra.run.smithi123.stderr:mgr not available, waiting (2/15)... 2024-04-12T21:37:50.637 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:50 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1995311971' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: Activating manager daemon smithi123.psjrfs 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: mgrmap e2: smithi123.psjrfs(active, starting, since 0.00253693s) 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:37:52.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:37:52.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' 2024-04-12T21:37:52.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' 2024-04-12T21:37:52.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:52 smithi123 ceph-mon[24842]: from='mgr.14100 172.21.15.123:0/2984049617' entity='mgr.smithi123.psjrfs' 2024-04-12T21:37:52.810 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:52.810 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-04-12T21:37:52.810 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsid": "c8b0510a-f914-11ee-bc8d-c7b262605968", 2024-04-12T21:37:52.810 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "health": { 2024-04-12T21:37:52.810 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "checks": {}, 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mutes": [] 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum": [ 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 0 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "smithi123" 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "monmap": { 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-12T21:37:52.811 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osdmap": { 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgmap": { 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-12T21:37:52.812 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsmap": { 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": false, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modules": [ 2024-04-12T21:37:52.813 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "iostat", 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "nfs", 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "restful" 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "servicemap": { 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modified": "2024-04-12T21:37:29.008157+0000", 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-12T21:37:52.814 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-04-12T21:37:53.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:53 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2712544139' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T21:37:53.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:53 smithi123 ceph-mon[24842]: mgrmap e3: smithi123.psjrfs(active, since 1.00574s) 2024-04-12T21:37:53.389 INFO:teuthology.orchestra.run.smithi123.stderr:mgr not available, waiting (3/15)... 2024-04-12T21:37:55.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:55 smithi123 ceph-mon[24842]: mgrmap e4: smithi123.psjrfs(active, since 2s) 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsid": "c8b0510a-f914-11ee-bc8d-c7b262605968", 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "health": { 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "checks": {}, 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mutes": [] 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum": [ 2024-04-12T21:37:56.051 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 0 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "smithi123" 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "monmap": { 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osdmap": { 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-12T21:37:56.052 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgmap": { 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:56.056 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsmap": { 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": true, 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modules": [ 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "iostat", 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "nfs", 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "restful" 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "servicemap": { 2024-04-12T21:37:56.057 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T21:37:56.058 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modified": "2024-04-12T21:37:29.008157+0000", 2024-04-12T21:37:56.058 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-04-12T21:37:56.058 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-04-12T21:37:56.058 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-12T21:37:56.058 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-04-12T21:37:56.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:56 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1421815492' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T21:37:56.481 INFO:teuthology.orchestra.run.smithi123.stderr:mgr is available 2024-04-12T21:37:57.064 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:57.064 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [global] 2024-04-12T21:37:57.064 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: fsid = c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:37:57.064 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-12T21:37:57.064 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-12T21:37:57.064 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-12T21:37:57.064 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-12T21:37:57.064 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:57.065 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [mgr] 2024-04-12T21:37:57.065 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-12T21:37:57.065 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-04-12T21:37:57.065 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [osd] 2024-04-12T21:37:57.065 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-12T21:37:57.065 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-12T21:37:57.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:57 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3446188329' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-12T21:37:57.601 INFO:teuthology.orchestra.run.smithi123.stderr:Enabling cephadm module... 2024-04-12T21:37:58.595 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:58 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/4134955911' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-12T21:37:59.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:59 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/4134955911' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-12T21:37:59.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:37:59 smithi123 ceph-mon[24842]: mgrmap e5: smithi123.psjrfs(active, since 6s) 2024-04-12T21:37:59.552 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-04-12T21:37:59.552 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-12T21:37:59.552 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": true, 2024-04-12T21:37:59.552 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "active_name": "smithi123.psjrfs", 2024-04-12T21:37:59.552 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-12T21:37:59.553 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-04-12T21:38:00.130 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for the mgr to restart... 2024-04-12T21:38:00.130 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mgr epoch 5... 2024-04-12T21:38:00.331 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:00 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2282826916' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-12T21:38:05.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: Active manager daemon smithi123.psjrfs restarted 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: Activating manager daemon smithi123.psjrfs 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: osdmap e2: 0 total, 0 up, 0 in 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: mgrmap e6: smithi123.psjrfs(active, starting, since 0.0525803s) 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:38:05.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:38:05.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:38:05.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:38:05.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:38:05.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:05.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:05 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:38:05.172 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-04-12T21:38:05.172 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-12T21:38:05.172 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "initialized": true 2024-04-12T21:38:05.172 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-04-12T21:38:05.717 INFO:teuthology.orchestra.run.smithi123.stderr:mgr epoch 5 is available 2024-04-12T21:38:05.718 INFO:teuthology.orchestra.run.smithi123.stderr:Setting orchestrator backend to cephadm... 2024-04-12T21:38:06.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:06 smithi123 ceph-mon[24842]: [12/Apr/2024:21:38:04] ENGINE Bus STARTING 2024-04-12T21:38:06.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:06 smithi123 ceph-mon[24842]: [12/Apr/2024:21:38:04] ENGINE Serving on https://172.21.15.123:7150 2024-04-12T21:38:06.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:06 smithi123 ceph-mon[24842]: [12/Apr/2024:21:38:04] ENGINE Bus STARTED 2024-04-12T21:38:06.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:06 smithi123 ceph-mon[24842]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-12T21:38:06.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:06 smithi123 ceph-mon[24842]: mgrmap e7: smithi123.psjrfs(active, since 1.05545s) 2024-04-12T21:38:06.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:06 smithi123 ceph-mon[24842]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-12T21:38:07.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:07 smithi123 ceph-mon[24842]: mgrmap e8: smithi123.psjrfs(active, since 2s) 2024-04-12T21:38:07.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:07 smithi123 ceph-mon[24842]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:07.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:07 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:07.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:07 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:38:07.476 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: value unchanged 2024-04-12T21:38:08.179 INFO:teuthology.orchestra.run.smithi123.stderr:Generating ssh key... 2024-04-12T21:38:08.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:08 smithi123 ceph-mon[24842]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:10.011 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:09 smithi123 ceph-mon[24842]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:10.011 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:09 smithi123 ceph-mon[24842]: Generating ssh key... 2024-04-12T21:38:10.011 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:09 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:10.011 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:09 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:10.011 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3t3j0oXZm83jutMHkh2sKVg0fwUzLr9SZQoez7hHNZW2GR3uwpHNKFLu6F+akOs9opvWFeRtRi/cLxTX4G+Y9DGKHrKA99LUv5bh1odXBahq7ZwzdlLyfzOO8+87YMRgHWHl5PuVQRZTf1NsrCSifd61u/wfx5wBkPOlBuUEanBnLvGmHHuus4mA3jjDzaz1fV8h9LU5WM2HMFo/RlS4QkKPXtUjlNliPNTcCtnistlVRmbJ+4TdrUE2zvrv+Rb1jTtrypHMmSrQdXQVPvu2i9Bf3pO1gptWJRWF0FBV1dXdW73Os7VEdaCfs4AGKT8dKOxPDEuKd0NJlD1PWou5+bbRYroMh3wDMtTWl4PdlRC4DD/cHXkBoaHknNFuVkNU3oCRtL2uJkGGQQdjnuwhHLrzsHhJB9myVsWDKQHzXPAjLiV2r31kU62tUmrcE25qpHP/aJsjX0jhlephUdp+r5v0rLamHy4m7Hf6Rt4+W8UUx6V4WPtc6yEcZ4xJQIxU= ceph-c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:38:10.582 INFO:teuthology.orchestra.run.smithi123.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-12T21:38:10.582 INFO:teuthology.orchestra.run.smithi123.stderr:Adding key to root@localhost authorized_keys... 2024-04-12T21:38:10.583 INFO:teuthology.orchestra.run.smithi123.stderr:Adding host smithi123... 2024-04-12T21:38:11.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:10 smithi123 ceph-mon[24842]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:12.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:11 smithi123 ceph-mon[24842]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi123", "addr": "172.21.15.123", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:12.200 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Added host 'smithi123' with addr '172.21.15.123' 2024-04-12T21:38:12.764 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying mon service with default placement... 2024-04-12T21:38:13.407 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:13 smithi123 ceph-mon[24842]: Deploying cephadm binary to smithi123 2024-04-12T21:38:13.407 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:13 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:13.407 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:13 smithi123 ceph-mon[24842]: Added host smithi123 2024-04-12T21:38:13.407 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:13 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:38:13.408 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-12T21:38:14.131 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying mgr service with default placement... 2024-04-12T21:38:14.667 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:14 smithi123 ceph-mon[24842]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:14.667 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:14 smithi123 ceph-mon[24842]: Saving service mon spec with placement count:5 2024-04-12T21:38:14.667 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:14 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:14.891 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-12T21:38:15.556 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying crash service with default placement... 2024-04-12T21:38:16.123 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:15 smithi123 ceph-mon[24842]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:16.123 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:15 smithi123 ceph-mon[24842]: Saving service mgr spec with placement count:2 2024-04-12T21:38:16.123 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:15 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:16.123 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-12T21:38:17.064 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying prometheus service with default placement... 2024-04-12T21:38:17.365 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:17 smithi123 ceph-mon[24842]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:17.365 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:17 smithi123 ceph-mon[24842]: Saving service crash spec with placement * 2024-04-12T21:38:17.365 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:17 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:17.365 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:17 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:17.663 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-12T21:38:18.220 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying grafana service with default placement... 2024-04-12T21:38:18.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:18 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:18.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:18 smithi123 ceph-mon[24842]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:18.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:18 smithi123 ceph-mon[24842]: Saving service prometheus spec with placement count:1 2024-04-12T21:38:18.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:18 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:18.848 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-12T21:38:19.411 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying node-exporter service with default placement... 2024-04-12T21:38:20.085 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-12T21:38:20.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:19 smithi123 ceph-mon[24842]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:20.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:19 smithi123 ceph-mon[24842]: Saving service grafana spec with placement count:1 2024-04-12T21:38:20.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:19 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:20.712 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying alertmanager service with default placement... 2024-04-12T21:38:21.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:21 smithi123 ceph-mon[24842]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:21.325 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:21 smithi123 ceph-mon[24842]: Saving service node-exporter spec with placement * 2024-04-12T21:38:21.325 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:21 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:21.363 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-12T21:38:22.217 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:22 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:22.217 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:22 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:22.218 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:22 smithi123 ceph-mon[24842]: from='mgr.14120 172.21.15.123:0/2116599065' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:23.532 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:23 smithi123 ceph-mon[24842]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:23.532 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:23 smithi123 ceph-mon[24842]: Saving service alertmanager spec with placement count:1 2024-04-12T21:38:23.533 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:23 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2568281199' entity='client.admin' 2024-04-12T21:38:24.968 INFO:teuthology.orchestra.run.smithi123.stderr:Enabling the dashboard module... 2024-04-12T21:38:25.092 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:24 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/717155945' entity='client.admin' 2024-04-12T21:38:25.092 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:24 smithi123 ceph-mon[24842]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:38:26.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:25 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1433946827' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-12T21:38:27.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:26 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1433946827' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-12T21:38:27.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:26 smithi123 ceph-mon[24842]: mgrmap e9: smithi123.psjrfs(active, since 21s) 2024-04-12T21:38:27.218 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-04-12T21:38:27.218 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-12T21:38:27.218 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": true, 2024-04-12T21:38:27.218 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "active_name": "smithi123.psjrfs", 2024-04-12T21:38:27.218 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-12T21:38:27.218 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-04-12T21:38:27.805 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for the mgr to restart... 2024-04-12T21:38:27.805 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mgr epoch 9... 2024-04-12T21:38:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:27 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/820392652' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-12T21:38:32.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: Active manager daemon smithi123.psjrfs restarted 2024-04-12T21:38:32.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: Activating manager daemon smithi123.psjrfs 2024-04-12T21:38:32.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: osdmap e3: 0 total, 0 up, 0 in 2024-04-12T21:38:32.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: mgrmap e10: smithi123.psjrfs(active, starting, since 0.0531076s) 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:38:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:32 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:32.752 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-04-12T21:38:32.752 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-12T21:38:32.752 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "initialized": true 2024-04-12T21:38:32.752 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-04-12T21:38:33.448 INFO:teuthology.orchestra.run.smithi123.stderr:mgr epoch 9 is available 2024-04-12T21:38:33.448 INFO:teuthology.orchestra.run.smithi123.stderr:Generating a dashboard self-signed certificate... 2024-04-12T21:38:34.012 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:33 smithi123 ceph-mon[24842]: [12/Apr/2024:21:38:31] ENGINE Bus STARTING 2024-04-12T21:38:34.012 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:33 smithi123 ceph-mon[24842]: [12/Apr/2024:21:38:32] ENGINE Serving on https://172.21.15.123:7150 2024-04-12T21:38:34.012 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:33 smithi123 ceph-mon[24842]: [12/Apr/2024:21:38:32] ENGINE Bus STARTED 2024-04-12T21:38:34.012 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:33 smithi123 ceph-mon[24842]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-12T21:38:34.012 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:33 smithi123 ceph-mon[24842]: mgrmap e11: smithi123.psjrfs(active, since 1.05549s) 2024-04-12T21:38:34.012 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:33 smithi123 ceph-mon[24842]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-12T21:38:34.288 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-12T21:38:34.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:34 smithi123 ceph-mon[24842]: mgrmap e12: smithi123.psjrfs(active, since 2s) 2024-04-12T21:38:34.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:34 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:34.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:34 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:34.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:34 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:34.846 INFO:teuthology.orchestra.run.smithi123.stderr:Creating initial admin user... 2024-04-12T21:38:35.799 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$.F10Xj/h1hJmFcM0Sxhm0OVJBbJTYnjxZ/3xdFlMvDQaxa0DP.g3y", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712957915, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-12T21:38:36.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:35 smithi123 ceph-mon[24842]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:36.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:35 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:36.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:35 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:36.531 INFO:teuthology.orchestra.run.smithi123.stderr:Fetching dashboard port number... 2024-04-12T21:38:37.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:36 smithi123 ceph-mon[24842]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:37.176 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 8443 2024-04-12T21:38:38.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:37 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1868682629' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-12T21:38:38.219 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: disabled 2024-04-12T21:38:38.226 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-04-12T21:38:38.227 INFO:teuthology.orchestra.run.smithi123.stderr:firewalld.service is not enabled 2024-04-12T21:38:38.227 INFO:teuthology.orchestra.run.smithi123.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-12T21:38:38.229 INFO:teuthology.orchestra.run.smithi123.stderr:Ceph Dashboard is now available at: 2024-04-12T21:38:38.229 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:38:38.229 INFO:teuthology.orchestra.run.smithi123.stderr: URL: https://smithi123.front.sepia.ceph.com:8443/ 2024-04-12T21:38:38.230 INFO:teuthology.orchestra.run.smithi123.stderr: User: admin 2024-04-12T21:38:38.230 INFO:teuthology.orchestra.run.smithi123.stderr: Password: tf9yzxfdmm 2024-04-12T21:38:38.230 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:38:38.230 INFO:teuthology.orchestra.run.smithi123.stderr:Enabling autotune for osd_memory_target 2024-04-12T21:38:39.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:38 smithi123 ceph-mon[24842]: mgrmap e13: smithi123.psjrfs(active, since 6s) 2024-04-12T21:38:40.251 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-12T21:38:40.854 INFO:teuthology.orchestra.run.smithi123.stderr:You can access the Ceph CLI with: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: ceph telemetry on 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr:For more information see: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T21:38:40.855 INFO:teuthology.orchestra.run.smithi123.stderr:Bootstrap complete. 2024-04-12T21:38:40.889 INFO:tasks.cephadm:Fetching config... 2024-04-12T21:38:40.890 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:38:40.890 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-12T21:38:40.906 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-12T21:38:40.906 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:38:40.906 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-12T21:38:40.961 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-12T21:38:40.961 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:38:40.961 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/keyring of=/dev/stdout 2024-04-12T21:38:41.028 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-12T21:38:41.028 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:38:41.028 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-12T21:38:41.083 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-12T21:38:41.083 DEBUG:teuthology.orchestra.run.smithi123:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3t3j0oXZm83jutMHkh2sKVg0fwUzLr9SZQoez7hHNZW2GR3uwpHNKFLu6F+akOs9opvWFeRtRi/cLxTX4G+Y9DGKHrKA99LUv5bh1odXBahq7ZwzdlLyfzOO8+87YMRgHWHl5PuVQRZTf1NsrCSifd61u/wfx5wBkPOlBuUEanBnLvGmHHuus4mA3jjDzaz1fV8h9LU5WM2HMFo/RlS4QkKPXtUjlNliPNTcCtnistlVRmbJ+4TdrUE2zvrv+Rb1jTtrypHMmSrQdXQVPvu2i9Bf3pO1gptWJRWF0FBV1dXdW73Os7VEdaCfs4AGKT8dKOxPDEuKd0NJlD1PWou5+bbRYroMh3wDMtTWl4PdlRC4DD/cHXkBoaHknNFuVkNU3oCRtL2uJkGGQQdjnuwhHLrzsHhJB9myVsWDKQHzXPAjLiV2r31kU62tUmrcE25qpHP/aJsjX0jhlephUdp+r5v0rLamHy4m7Hf6Rt4+W8UUx6V4WPtc6yEcZ4xJQIxU= ceph-c8b0510a-f914-11ee-bc8d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-12T21:38:41.171 INFO:teuthology.orchestra.run.smithi123.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3t3j0oXZm83jutMHkh2sKVg0fwUzLr9SZQoez7hHNZW2GR3uwpHNKFLu6F+akOs9opvWFeRtRi/cLxTX4G+Y9DGKHrKA99LUv5bh1odXBahq7ZwzdlLyfzOO8+87YMRgHWHl5PuVQRZTf1NsrCSifd61u/wfx5wBkPOlBuUEanBnLvGmHHuus4mA3jjDzaz1fV8h9LU5WM2HMFo/RlS4QkKPXtUjlNliPNTcCtnistlVRmbJ+4TdrUE2zvrv+Rb1jTtrypHMmSrQdXQVPvu2i9Bf3pO1gptWJRWF0FBV1dXdW73Os7VEdaCfs4AGKT8dKOxPDEuKd0NJlD1PWou5+bbRYroMh3wDMtTWl4PdlRC4DD/cHXkBoaHknNFuVkNU3oCRtL2uJkGGQQdjnuwhHLrzsHhJB9myVsWDKQHzXPAjLiV2r31kU62tUmrcE25qpHP/aJsjX0jhlephUdp+r5v0rLamHy4m7Hf6Rt4+W8UUx6V4WPtc6yEcZ4xJQIxU= ceph-c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:38:41.190 DEBUG:teuthology.orchestra.run.smithi177:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3t3j0oXZm83jutMHkh2sKVg0fwUzLr9SZQoez7hHNZW2GR3uwpHNKFLu6F+akOs9opvWFeRtRi/cLxTX4G+Y9DGKHrKA99LUv5bh1odXBahq7ZwzdlLyfzOO8+87YMRgHWHl5PuVQRZTf1NsrCSifd61u/wfx5wBkPOlBuUEanBnLvGmHHuus4mA3jjDzaz1fV8h9LU5WM2HMFo/RlS4QkKPXtUjlNliPNTcCtnistlVRmbJ+4TdrUE2zvrv+Rb1jTtrypHMmSrQdXQVPvu2i9Bf3pO1gptWJRWF0FBV1dXdW73Os7VEdaCfs4AGKT8dKOxPDEuKd0NJlD1PWou5+bbRYroMh3wDMtTWl4PdlRC4DD/cHXkBoaHknNFuVkNU3oCRtL2uJkGGQQdjnuwhHLrzsHhJB9myVsWDKQHzXPAjLiV2r31kU62tUmrcE25qpHP/aJsjX0jhlephUdp+r5v0rLamHy4m7Hf6Rt4+W8UUx6V4WPtc6yEcZ4xJQIxU= ceph-c8b0510a-f914-11ee-bc8d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-12T21:38:41.238 INFO:teuthology.orchestra.run.smithi177.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3t3j0oXZm83jutMHkh2sKVg0fwUzLr9SZQoez7hHNZW2GR3uwpHNKFLu6F+akOs9opvWFeRtRi/cLxTX4G+Y9DGKHrKA99LUv5bh1odXBahq7ZwzdlLyfzOO8+87YMRgHWHl5PuVQRZTf1NsrCSifd61u/wfx5wBkPOlBuUEanBnLvGmHHuus4mA3jjDzaz1fV8h9LU5WM2HMFo/RlS4QkKPXtUjlNliPNTcCtnistlVRmbJ+4TdrUE2zvrv+Rb1jTtrypHMmSrQdXQVPvu2i9Bf3pO1gptWJRWF0FBV1dXdW73Os7VEdaCfs4AGKT8dKOxPDEuKd0NJlD1PWou5+bbRYroMh3wDMtTWl4PdlRC4DD/cHXkBoaHknNFuVkNU3oCRtL2uJkGGQQdjnuwhHLrzsHhJB9myVsWDKQHzXPAjLiV2r31kU62tUmrcE25qpHP/aJsjX0jhlephUdp+r5v0rLamHy4m7Hf6Rt4+W8UUx6V4WPtc6yEcZ4xJQIxU= ceph-c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:38:41.253 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-12T21:38:41.277 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:41 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2334091309' entity='client.admin' 2024-04-12T21:38:44.021 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:43 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:44.021 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:43 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:38:44.021 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:43 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:44.021 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:43 smithi123 ceph-mon[24842]: Deploying daemon alertmanager.smithi123 on smithi123 2024-04-12T21:38:45.203 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-12T21:38:45.203 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-12T21:38:45.639 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:45 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2110835313' entity='client.admin' 2024-04-12T21:38:49.118 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:48 smithi123 ceph-mon[24842]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:49.118 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:48 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:49.142 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi177 2024-04-12T21:38:49.142 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-04-12T21:38:49.142 DEBUG:teuthology.orchestra.run.smithi177:> dd of=/etc/ceph/ceph.conf 2024-04-12T21:38:49.161 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-04-12T21:38:49.162 DEBUG:teuthology.orchestra.run.smithi177:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:38:49.218 INFO:tasks.cephadm:Adding host smithi177 to orchestrator... 2024-04-12T21:38:49.219 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch host add smithi177 2024-04-12T21:38:52.672 INFO:teuthology.orchestra.run.smithi123.stdout:Added host 'smithi177' with addr '172.21.15.177' 2024-04-12T21:38:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:52 smithi123 ceph-mon[24842]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi177", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:38:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:52 smithi123 ceph-mon[24842]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:38:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:52 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:54.075 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:53 smithi123 ceph-mon[24842]: Deploying cephadm binary to smithi177 2024-04-12T21:38:54.075 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:53 smithi123 ceph-mon[24842]: Added host smithi177 2024-04-12T21:38:54.579 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch host ls --format=json 2024-04-12T21:38:54.887 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:54 smithi123 ceph-mon[24842]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:38:57.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:56 smithi123 ceph-mon[24842]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:38:57.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:56 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:57.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:56 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:38:57.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:56 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-12T21:38:57.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:56 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:38:57.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:56 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:38:57.124 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:38:57.124 INFO:teuthology.orchestra.run.smithi123.stdout:[{"addr": "172.21.15.123", "hostname": "smithi123", "labels": [], "status": ""}, {"addr": "172.21.15.177", "hostname": "smithi177", "labels": [], "status": ""}] 2024-04-12T21:38:57.738 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-12T21:38:57.738 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd crush tunables default 2024-04-12T21:38:58.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:57 smithi123 ceph-mon[24842]: Deploying daemon crash.smithi123 on smithi123 2024-04-12T21:38:59.157 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:58 smithi123 ceph-mon[24842]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:38:59.157 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:38:58 smithi123 ceph-mon[24842]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:00.340 INFO:teuthology.orchestra.run.smithi123.stderr:adjusted tunables profile to default 2024-04-12T21:39:00.397 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:00 smithi123 ceph-mon[24842]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:00.397 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:00 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2780397713' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-12T21:39:01.098 INFO:tasks.cephadm:Adding mon.smithi123 on smithi123 2024-04-12T21:39:01.098 INFO:tasks.cephadm:Adding mon.smithi177 on smithi177 2024-04-12T21:39:01.098 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch apply mon '2;smithi123:172.21.15.123=smithi123;smithi177:172.21.15.177=smithi177' 2024-04-12T21:39:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2780397713' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-12T21:39:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T21:39:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T21:39:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T21:39:01.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:01.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:01 smithi123 ceph-mon[24842]: Deploying daemon grafana.smithi123 on smithi123 2024-04-12T21:39:02.749 INFO:teuthology.orchestra.run.smithi177.stdout:Scheduled mon update... 2024-04-12T21:39:03.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:02 smithi123 ceph-mon[24842]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:03.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:02 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:03.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:02 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:03.308 DEBUG:teuthology.orchestra.run.smithi177:mon.smithi177> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi177.service 2024-04-12T21:39:03.310 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:03.311 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:04.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:03 smithi123 ceph-mon[24842]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi123:172.21.15.123=smithi123;smithi177:172.21.15.177=smithi177", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:39:04.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:03 smithi123 ceph-mon[24842]: Saving service mon spec with placement smithi123:172.21.15.123=smithi123;smithi177:172.21.15.177=smithi177;count:2 2024-04-12T21:39:05.147 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:05.147 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:05.149 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:05.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:04 smithi123 ceph-mon[24842]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:06.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:05 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/2722348927' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:06.728 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:06.729 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:07.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:06 smithi123 ceph-mon[24842]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:08.527 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:08.527 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:08.529 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:09.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:08 smithi123 ceph-mon[24842]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:09.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:08 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/2177882129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:10.063 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:10.063 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:10.667 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:10 smithi123 ceph-mon[24842]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:11.862 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:11.862 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:11.863 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:13.091 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:12 smithi123 ceph-mon[24842]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:13.091 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:12 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/14909681' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:13.382 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:13.383 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:15.089 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:14 smithi123 ceph-mon[24842]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:15.089 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:14 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:15.089 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:14 smithi123 ceph-mon[24842]: Deploying daemon node-exporter.smithi123 on smithi123 2024-04-12T21:39:15.224 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:15.225 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:15.226 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:16.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:15 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/2631690280' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:16.770 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:16.770 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:16.917 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:16 smithi123 ceph-mon[24842]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:16.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:16 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:16.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:16 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T21:39:16.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:16 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:18.531 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:18.531 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:18.532 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:18.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:18 smithi123 ceph-mon[24842]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T21:39:18.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:18 smithi123 ceph-mon[24842]: mgrmap e14: smithi123.psjrfs(active, since 45s) 2024-04-12T21:39:18.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:18 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/780334825' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:20.022 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:20.022 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:21.667 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:21.667 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:21.669 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:21.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:21 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/3939392634' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:23.191 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:23.192 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:23.793 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:23 smithi123 ceph-mon[24842]: Active manager daemon smithi123.psjrfs restarted 2024-04-12T21:39:23.793 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:23 smithi123 ceph-mon[24842]: Activating manager daemon smithi123.psjrfs 2024-04-12T21:39:23.793 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:23 smithi123 ceph-mon[24842]: osdmap e5: 0 total, 0 up, 0 in 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: mgrmap e15: smithi123.psjrfs(active, starting, since 0.055312s) 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:39:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:24.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:39:24.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:24.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:39:24.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:39:24.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:24.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:25.059 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:25.059 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:25.061 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:25.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:25 smithi123 ceph-mon[24842]: [12/Apr/2024:21:39:23] ENGINE Bus STARTING 2024-04-12T21:39:25.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:25 smithi123 ceph-mon[24842]: [12/Apr/2024:21:39:24] ENGINE Serving on https://172.21.15.123:7150 2024-04-12T21:39:25.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:25 smithi123 ceph-mon[24842]: [12/Apr/2024:21:39:24] ENGINE Bus STARTED 2024-04-12T21:39:25.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:25 smithi123 ceph-mon[24842]: mgrmap e16: smithi123.psjrfs(active, since 1.06581s) 2024-04-12T21:39:25.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:25 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:25.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:25 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/1878634240' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:26.622 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:26.622 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:26.917 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:26 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:26.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:26 smithi123 ceph-mon[24842]: mgrmap e17: smithi123.psjrfs(active, since 2s) 2024-04-12T21:39:28.039 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:28.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:28.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:39:28.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:27 smithi123 ceph-mon[24842]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:39:29.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:29 smithi123 ceph-mon[24842]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:39:29.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:29 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:29.656 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:29.657 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:29.658 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd/host:smithi177", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: Deploying daemon crash.smithi177 on smithi177 2024-04-12T21:39:30.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:30 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/616416702' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:31.989 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:31.990 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:33.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:32 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:33.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:32 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:39:33.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:32 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T21:39:33.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:32 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:39:33.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:32 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:33.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:32 smithi123 ceph-mon[24842]: Deploying daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:39:34.374 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:34.374 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":1,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:37:26.292641Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T21:39:34.375 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 1 2024-04-12T21:39:34.917 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:34.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:34 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/453177153' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:36.070 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T21:39:36.070 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T21:39:36.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:36 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:36.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:36 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:39:36.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:36 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:36.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:36 smithi123 ceph-mon[24842]: Deploying daemon mon.smithi177 on smithi177 2024-04-12T21:39:38.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:38 smithi177 systemd[1]: Starting Ceph mon.smithi177 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:39:39.179 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 podman[26845]: 2024-04-12 21:39:38.952089381 +0000 UTC m=+0.020452075 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-12T21:39:39.470 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 podman[26845]: 2024-04-12 21:39:39.285065028 +0000 UTC m=+0.353427704 container create ae9bbb3ebc295646936f21f06a8ebbc63123b4ec3785a080e57ac456af785a37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, vendor=Red Hat, Inc., GIT_CLEAN=True, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=) 2024-04-12T21:39:39.736 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 podman[26845]: 2024-04-12 21:39:39.713830248 +0000 UTC m=+0.782192947 container init ae9bbb3ebc295646936f21f06a8ebbc63123b4ec3785a080e57ac456af785a37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, version=8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.expose-services=, ceph=True, GIT_BRANCH=HEAD) 2024-04-12T21:39:39.736 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 podman[26845]: 2024-04-12 21:39:39.717056701 +0000 UTC m=+0.785419388 container start ae9bbb3ebc295646936f21f06a8ebbc63123b4ec3785a080e57ac456af785a37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, RELEASE=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, version=8, vcs-type=git, release=754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-12T21:39:39.917 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.014 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pidfile_write: ignore empty --pid-file 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: load: jerasure load: lrc 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: RocksDB version: 6.15.5 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Compile date Apr 18 2022 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: DB SUMMARY 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: DB Session ID: 3JG8I33K5DJ2R8HZ5S7N 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: CURRENT file: CURRENT 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T21:39:40.015 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi177/store.db dir, Total Num: 0, files: 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi177/store.db: 000004.log size: 511 ; 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.error_if_exists: 0 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.create_if_missing: 0 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.paranoid_checks: 1 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.env: 0x55a147cc6860 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.fs: Posix File System 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.info_log: 0x55a148dedfa0 2024-04-12T21:39:40.016 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.statistics: (nil) 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.use_fsync: 0 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_log_file_size: 0 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.allow_fallocate: 1 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T21:39:40.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.use_direct_reads: 0 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.db_log_dir: 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi177/store.db 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T21:39:40.018 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.write_buffer_manager: 0x55a148ede2d0 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T21:39:40.019 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.unordered_write: 0 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.row_cache: None 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.wal_filter: None 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.preserve_deletes: 0 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.two_write_queues: 0 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T21:39:40.020 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.atomic_flush: 0 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.log_readahead_size: 0 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T21:39:40.021 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_background_jobs: 2 2024-04-12T21:39:40.023 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_background_compactions: -1 2024-04-12T21:39:40.023 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_subcompactions: 1 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_open_files: -1 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T21:39:40.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_background_flushes: -1 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Compression algorithms supported: 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: kZSTD supported: 0 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: kXpressCompression supported: 0 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: kLZ4Compression supported: 1 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: kBZip2Compression supported: 0 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: kZlibCompression supported: 1 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: kSnappyCompression supported: 1 2024-04-12T21:39:40.025 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi177/store.db/MANIFEST-000003 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.merge_operator: 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_filter: None 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a148db9d00) 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: cache_index_and_filter_blocks: 1 2024-04-12T21:39:40.026 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: pin_top_level_index_and_filter: 1 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: index_type: 0 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: data_block_index_type: 0 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: index_shortening: 1 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: hash_index_allow_collision: 1 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: checksum: 1 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: no_block_cache: 0 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_cache: 0x55a148e24170 2024-04-12T21:39:40.027 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_cache_name: BinnedLRUCache 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_cache_options: 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: capacity : 536870912 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: num_shard_bits : 4 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: strict_capacity_limit : 0 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: high_pri_pool_ratio: 0.000 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_cache_compressed: (nil) 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: persistent_cache: (nil) 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_size: 4096 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_size_deviation: 10 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_restart_interval: 16 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: index_block_restart_interval: 1 2024-04-12T21:39:40.028 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: metadata_block_size: 4096 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: partition_filters: 0 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: use_delta_encoding: 1 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: whole_key_filtering: 1 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: verify_compression: 0 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: read_amp_bytes_per_bit: 0 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: format_version: 4 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: enable_index_compression: 1 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_align: 0 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T21:39:40.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compression: NoCompression 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.num_levels: 7 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T21:39:40.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T21:39:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T21:39:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.arena_block_size: 4194304 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T21:39:40.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T21:39:40.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T21:39:40.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T21:39:40.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T21:39:40.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.table_properties_collectors: 2024-04-12T21:39:40.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.inplace_update_support: 0 2024-04-12T21:39:40.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T21:39:40.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T21:39:40.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.bloom_locality: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.max_successive_merges: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.ttl: 2592000 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.enable_blob_files: false 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.min_blob_size: 0 2024-04-12T21:39:40.035 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi177/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712957979743704, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-12T21:39:40.036 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712957979744133, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712957979, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "93fb56e4-d4a5-4bee-b106-6b8efbf3c9b9", "db_session_id": "3JG8I33K5DJ2R8HZ5S7N"}} 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712957979744539, "job": 1, "event": "recovery_finished"} 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi177/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55a148e0aa80 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: DB pointer 0x55a148e1a000 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** DB Stats ** 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T21:39:40.037 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** Compaction Stats [default] ** 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-12T21:39:40.038 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** Compaction Stats [default] ** 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative compaction: 0.00 GB write, 0.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:39:40.039 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval compaction: 0.00 GB write, 0.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** Compaction Stats [default] ** 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-12T21:39:40.040 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** Compaction Stats [default] ** 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T21:39:40.041 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative compaction: 0.00 GB write, 0.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177 does not exist in monmap, will attempt to join an existing cluster 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: using public_addr v2:172.21.15.177:0/0 -> [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: starting mon.smithi177 rank -1 at public addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] at bind addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi177 fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(???) e0 preinit fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).mds e1 new map 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).mds e1 print_map 2024-04-12T21:39:40.042 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: e1 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: legacy client fscid: -1 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: No filesystems configured 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-04-12T21:39:40.043 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Deploying daemon crash.smithi123 on smithi123 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.044 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2780397713' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2780397713' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Deploying daemon grafana.smithi123 on smithi123 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.045 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi123:172.21.15.123=smithi123;smithi177:172.21.15.177=smithi177", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Saving service mon spec with placement smithi123:172.21.15.123=smithi123;smithi177:172.21.15.177=smithi177;count:2 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/2722348927' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/2177882129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/14909681' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.046 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Deploying daemon node-exporter.smithi123 on smithi123 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/2631690280' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14162 172.21.15.123:0/3539598249' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mgrmap e14: smithi123.psjrfs(active, since 45s) 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/780334825' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/3939392634' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.047 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Active manager daemon smithi123.psjrfs restarted 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Activating manager daemon smithi123.psjrfs 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: osdmap e5: 0 total, 0 up, 0 in 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mgrmap e15: smithi123.psjrfs(active, starting, since 0.055312s) 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.048 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: [12/Apr/2024:21:39:23] ENGINE Bus STARTING 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: [12/Apr/2024:21:39:24] ENGINE Serving on https://172.21.15.123:7150 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: [12/Apr/2024:21:39:24] ENGINE Bus STARTED 2024-04-12T21:39:40.049 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mgrmap e16: smithi123.psjrfs(active, since 1.06581s) 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/1878634240' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mgrmap e17: smithi123.psjrfs(active, since 2s) 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.050 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd/host:smithi177", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Deploying daemon crash.smithi177 on smithi177 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/616416702' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:39:40.051 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Deploying daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/453177153' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: Deploying daemon mon.smithi177 on smithi177 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-04-12T21:39:40.052 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 ceph-mon[26903]: mon.smithi177@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-12T21:39:40.053 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 bash[26845]: ae9bbb3ebc295646936f21f06a8ebbc63123b4ec3785a080e57ac456af785a37 2024-04-12T21:39:40.053 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:39 smithi177 systemd[1]: Started Ceph mon.smithi177 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:39:44.785 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-12T21:39:44.786 INFO:teuthology.orchestra.run.smithi177.stdout:{"epoch":2,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","modified":"2024-04-12T21:39:39.773252Z","created":"2024-04-12T21:37:26.292641Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi177","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:3300","nonce":0},{"type":"v1","addr":"172.21.15.177:6789","nonce":0}]},"addr":"172.21.15.177:6789/0","public_addr":"172.21.15.177:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-12T21:39:44.788 INFO:teuthology.orchestra.run.smithi177.stderr:dumped monmap epoch 2 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: mon.smithi123 calling monitor election 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/4202579203' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/608018335' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: mon.smithi177 calling monitor election 2024-04-12T21:39:45.083 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: mon.smithi123 is new leader, mons smithi123,smithi177 in quorum (ranks 0,1) 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: monmap e2: 2 mons at {smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],smithi177=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0]} 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: fsmap 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: osdmap e5: 0 total, 0 up, 0 in 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: mgrmap e17: smithi123.psjrfs(active, since 21s) 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza started 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/608018335' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: overall HEALTH_OK 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/608018335' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:39:45.084 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/608018335' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:39:45.085 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:45.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:39:45.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: mon.smithi123 calling monitor election 2024-04-12T21:39:45.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/4202579203' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T21:39:45.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/608018335' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: mon.smithi177 calling monitor election 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: mon.smithi123 is new leader, mons smithi123,smithi177 in quorum (ranks 0,1) 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: monmap e2: 2 mons at {smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],smithi177=[v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0]} 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: fsmap 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: osdmap e5: 0 total, 0 up, 0 in 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: mgrmap e17: smithi123.psjrfs(active, since 21s) 2024-04-12T21:39:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: Standby manager daemon smithi177.whybza started 2024-04-12T21:39:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/608018335' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:39:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: overall HEALTH_OK 2024-04-12T21:39:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/608018335' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:39:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/608018335' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:39:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:45.388 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-12T21:39:45.388 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph config generate-minimal-conf 2024-04-12T21:39:46.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:45 smithi123 ceph-mon[24842]: Deploying daemon node-exporter.smithi177 on smithi177 2024-04-12T21:39:46.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:45 smithi123 ceph-mon[24842]: mgrmap e18: smithi123.psjrfs(active, since 21s), standbys: smithi177.whybza 2024-04-12T21:39:46.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:45 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi177.whybza", "id": "smithi177.whybza"}]: dispatch 2024-04-12T21:39:46.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:45 smithi123 ceph-mon[24842]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:46.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:45 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:46.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:45 smithi177 ceph-mon[26903]: Deploying daemon node-exporter.smithi177 on smithi177 2024-04-12T21:39:46.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:45 smithi177 ceph-mon[26903]: mgrmap e18: smithi123.psjrfs(active, since 21s), standbys: smithi177.whybza 2024-04-12T21:39:46.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:45 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi177.whybza", "id": "smithi177.whybza"}]: dispatch 2024-04-12T21:39:46.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:45 smithi177 ceph-mon[26903]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:46.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:45 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:39:47.189 INFO:teuthology.orchestra.run.smithi123.stdout:# minimal ceph.conf for c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:39:47.189 INFO:teuthology.orchestra.run.smithi123.stdout:[global] 2024-04-12T21:39:47.189 INFO:teuthology.orchestra.run.smithi123.stdout: fsid = c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:39:47.189 INFO:teuthology.orchestra.run.smithi123.stdout: mon_host = [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] 2024-04-12T21:39:47.447 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:47 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2123371439' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:47.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:47 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2123371439' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:47.820 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-12T21:39:47.820 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:39:47.821 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.conf 2024-04-12T21:39:47.850 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:39:47.850 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:39:47.917 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-04-12T21:39:47.917 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/ceph/ceph.conf 2024-04-12T21:39:47.949 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-04-12T21:39:47.949 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:39:48.016 INFO:tasks.cephadm:Deploying OSDs... 2024-04-12T21:39:48.016 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:39:48.016 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/scratch_devs of=/dev/stdout 2024-04-12T21:39:48.031 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-12T21:39:48.031 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_1 2024-04-12T21:39:48.085 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-12T21:39:48.086 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T21:39:48.086 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-12T21:39:48.086 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T21:39:48.086 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T21:39:48.086 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-04-12 21:39:14.451383565 +0000 2024-04-12T21:39:48.086 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-04-12 21:36:24.021524104 +0000 2024-04-12T21:39:48.086 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-04-12 21:36:24.021524104 +0000 2024-04-12T21:39:48.086 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-04-12 21:36:24.021524104 +0000 2024-04-12T21:39:48.086 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-12T21:39:48.151 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-04-12T21:39:48.151 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-04-12T21:39:48.151 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000196882 s, 2.6 MB/s 2024-04-12T21:39:48.152 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-12T21:39:48.207 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_2 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-04-12 21:39:14.451383565 +0000 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-04-12 21:36:24.360515604 +0000 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-04-12 21:36:24.360515604 +0000 2024-04-12T21:39:48.262 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-04-12 21:36:24.360515604 +0000 2024-04-12T21:39:48.263 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-12T21:39:48.327 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-04-12T21:39:48.327 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-04-12T21:39:48.327 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000236701 s, 2.2 MB/s 2024-04-12T21:39:48.328 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-12T21:39:48.384 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_3 2024-04-12T21:39:48.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:48 smithi123 ceph-mon[24842]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:48.431 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-12T21:39:48.431 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T21:39:48.431 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-04-12T21:39:48.431 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T21:39:48.432 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T21:39:48.432 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-04-12 21:39:14.452383540 +0000 2024-04-12T21:39:48.432 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-04-12 21:36:24.693507255 +0000 2024-04-12T21:39:48.432 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-04-12 21:36:24.693507255 +0000 2024-04-12T21:39:48.432 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-04-12 21:36:24.693507255 +0000 2024-04-12T21:39:48.432 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-12T21:39:48.496 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-04-12T21:39:48.496 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-04-12T21:39:48.496 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000223705 s, 2.3 MB/s 2024-04-12T21:39:48.498 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-12T21:39:48.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:48 smithi177 ceph-mon[26903]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:48.553 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_4 2024-04-12T21:39:48.607 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-12T21:39:48.608 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T21:39:48.608 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-12T21:39:48.608 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T21:39:48.608 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T21:39:48.608 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-04-12 21:39:14.452383540 +0000 2024-04-12T21:39:48.608 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-04-12 21:36:25.026498906 +0000 2024-04-12T21:39:48.608 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-04-12 21:36:25.026498906 +0000 2024-04-12T21:39:48.608 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-04-12 21:36:25.026498906 +0000 2024-04-12T21:39:48.608 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-12T21:39:48.674 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-04-12T21:39:48.674 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-04-12T21:39:48.674 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000206004 s, 2.5 MB/s 2024-04-12T21:39:48.675 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-12T21:39:48.731 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-04-12T21:39:48.731 DEBUG:teuthology.orchestra.run.smithi177:> dd if=/scratch_devs of=/dev/stdout 2024-04-12T21:39:48.747 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-12T21:39:48.747 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_1 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2024-04-12 21:39:45.773759231 +0000 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2024-04-12 21:36:31.429393131 +0000 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2024-04-12 21:36:31.429393131 +0000 2024-04-12T21:39:48.803 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: 2024-04-12 21:36:31.429393131 +0000 2024-04-12T21:39:48.804 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-12T21:39:48.869 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2024-04-12T21:39:48.869 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2024-04-12T21:39:48.869 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000245835 s, 2.1 MB/s 2024-04-12T21:39:48.870 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-12T21:39:48.925 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_2 2024-04-12T21:39:48.980 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-12T21:39:48.980 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T21:39:48.981 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-12T21:39:48.981 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T21:39:48.981 INFO:teuthology.orchestra.run.smithi177.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T21:39:48.981 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2024-04-12 21:39:45.773759231 +0000 2024-04-12T21:39:48.981 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2024-04-12 21:36:31.776383613 +0000 2024-04-12T21:39:48.981 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2024-04-12 21:36:31.776383613 +0000 2024-04-12T21:39:48.981 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: 2024-04-12 21:36:31.776383613 +0000 2024-04-12T21:39:48.981 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-12T21:39:49.046 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2024-04-12T21:39:49.046 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2024-04-12T21:39:49.046 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000255221 s, 2.0 MB/s 2024-04-12T21:39:49.047 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-12T21:39:49.103 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_3 2024-04-12T21:39:49.157 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-12T21:39:49.157 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T21:39:49.157 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-12T21:39:49.158 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T21:39:49.158 INFO:teuthology.orchestra.run.smithi177.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T21:39:49.158 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2024-04-12 21:39:45.773759231 +0000 2024-04-12T21:39:49.158 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2024-04-12 21:36:32.117374260 +0000 2024-04-12T21:39:49.158 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2024-04-12 21:36:32.117374260 +0000 2024-04-12T21:39:49.158 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: 2024-04-12 21:36:32.117374260 +0000 2024-04-12T21:39:49.158 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-12T21:39:49.223 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2024-04-12T21:39:49.223 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2024-04-12T21:39:49.223 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000264662 s, 1.9 MB/s 2024-04-12T21:39:49.224 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-12T21:39:49.280 DEBUG:teuthology.orchestra.run.smithi177:> stat /dev/vg_nvme/lv_4 2024-04-12T21:39:49.334 INFO:teuthology.orchestra.run.smithi177.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-12T21:39:49.334 INFO:teuthology.orchestra.run.smithi177.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T21:39:49.335 INFO:teuthology.orchestra.run.smithi177.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-12T21:39:49.335 INFO:teuthology.orchestra.run.smithi177.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T21:39:49.335 INFO:teuthology.orchestra.run.smithi177.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T21:39:49.335 INFO:teuthology.orchestra.run.smithi177.stdout:Access: 2024-04-12 21:39:45.774759204 +0000 2024-04-12T21:39:49.335 INFO:teuthology.orchestra.run.smithi177.stdout:Modify: 2024-04-12 21:36:32.443365318 +0000 2024-04-12T21:39:49.335 INFO:teuthology.orchestra.run.smithi177.stdout:Change: 2024-04-12 21:36:32.443365318 +0000 2024-04-12T21:39:49.335 INFO:teuthology.orchestra.run.smithi177.stdout: Birth: 2024-04-12 21:36:32.443365318 +0000 2024-04-12T21:39:49.335 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-12T21:39:49.400 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records in 2024-04-12T21:39:49.401 INFO:teuthology.orchestra.run.smithi177.stderr:1+0 records out 2024-04-12T21:39:49.401 INFO:teuthology.orchestra.run.smithi177.stderr:512 bytes copied, 0.000217618 s, 2.4 MB/s 2024-04-12T21:39:49.402 DEBUG:teuthology.orchestra.run.smithi177:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-12T21:39:49.457 INFO:tasks.cephadm:Deploying osd.0 on smithi123 with /dev/vg_nvme/lv_4... 2024-04-12T21:39:49.457 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-12T21:39:49.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:49.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:49.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:49 smithi123 ceph-mon[24842]: Deploying daemon prometheus.smithi123 on smithi123 2024-04-12T21:39:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:49 smithi177 ceph-mon[26903]: Deploying daemon prometheus.smithi123 on smithi123 2024-04-12T21:39:50.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:50 smithi177 ceph-mon[26903]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:50.854 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:50 smithi123 ceph-mon[24842]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:52.115 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-12T21:39:52.115 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-12T21:39:52.115 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2024-04-12T21:39:52.115 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2024-04-12T21:39:52.115 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135198 s, 776 MB/s 2024-04-12T21:39:52.115 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2024-04-12T21:39:52.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:52 smithi177 ceph-mon[26903]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:52 smithi123 ceph-mon[24842]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:53.134 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi123:vg_nvme/lv_4 2024-04-12T21:39:54.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:54 smithi177 ceph-mon[26903]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:54.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:54 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:54.917 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:54 smithi123 ceph-mon[24842]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:54.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:54 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:39:56.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:56 smithi177 ceph-mon[26903]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:56.890 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:56 smithi123 ceph-mon[24842]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:58.828 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:58 smithi123 ceph-mon[24842]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:58.828 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:58 smithi123 ceph-mon[24842]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:39:58.828 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:58 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:39:58.828 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:58 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:39:58.828 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:39:58 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:39:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:58 smithi177 ceph-mon[26903]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:39:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:58 smithi177 ceph-mon[26903]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:39:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:58 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:39:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:58 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:39:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:39:58 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:00.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:00 smithi123 ceph-mon[24842]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:00.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:00 smithi123 ceph-mon[24842]: overall HEALTH_OK 2024-04-12T21:40:00.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:00 smithi177 ceph-mon[26903]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:00.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:00 smithi177 ceph-mon[26903]: overall HEALTH_OK 2024-04-12T21:40:02.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:02 smithi177 ceph-mon[26903]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:02.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:02 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:02.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:02 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:40:02.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:02 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:02.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:02 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:40:02.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:02 smithi123 ceph-mon[24842]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:02.791 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:02 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:02.791 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:02 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:40:02.791 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:02 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:02.791 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:02 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:40:03.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:03 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1588818152' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb3f406e-c2cc-4356-8eb5-e9b82ac770b8"}]: dispatch 2024-04-12T21:40:03.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:03 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1588818152' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb3f406e-c2cc-4356-8eb5-e9b82ac770b8"}]': finished 2024-04-12T21:40:03.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:03 smithi123 ceph-mon[24842]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T21:40:03.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:03.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:03.981 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:03 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1588818152' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb3f406e-c2cc-4356-8eb5-e9b82ac770b8"}]: dispatch 2024-04-12T21:40:03.981 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:03 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1588818152' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb3f406e-c2cc-4356-8eb5-e9b82ac770b8"}]': finished 2024-04-12T21:40:03.981 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:03 smithi177 ceph-mon[26903]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T21:40:03.981 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:03.981 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:04.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:04 smithi123 ceph-mon[24842]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:04.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:04 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3923930148' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:40:04.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:04.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:04 smithi177 ceph-mon[26903]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:04 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3923930148' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:40:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:05.831 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:05 smithi123 ceph-mon[24842]: Updating smithi177:/etc/ceph/ceph.conf 2024-04-12T21:40:05.831 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:05 smithi123 ceph-mon[24842]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:40:06.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:05 smithi177 ceph-mon[26903]: Updating smithi177:/etc/ceph/ceph.conf 2024-04-12T21:40:06.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:05 smithi177 ceph-mon[26903]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:40:06.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:06 smithi123 ceph-mon[24842]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:06 smithi177 ceph-mon[26903]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: Reconfiguring mon.smithi123 (unknown last config time)... 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:08.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: Reconfiguring daemon mon.smithi123 on smithi123 2024-04-12T21:40:08.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:07 smithi123 ceph-mon[24842]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:08.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:08.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:40:08.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:08.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:08.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: Reconfiguring mon.smithi123 (unknown last config time)... 2024-04-12T21:40:08.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:40:08.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:40:08.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: Reconfiguring daemon mon.smithi123 on smithi123 2024-04-12T21:40:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:07 smithi177 ceph-mon[26903]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:10.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:10.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:10 smithi177 ceph-mon[26903]: Reconfiguring mgr.smithi123.psjrfs (unknown last config time)... 2024-04-12T21:40:10.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.psjrfs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:40:10.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:40:10.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:10.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:10 smithi177 ceph-mon[26903]: Reconfiguring daemon mgr.smithi123.psjrfs on smithi123 2024-04-12T21:40:10.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:10 smithi177 ceph-mon[26903]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:10 smithi123 ceph-mon[24842]: Reconfiguring mgr.smithi123.psjrfs (unknown last config time)... 2024-04-12T21:40:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.psjrfs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:40:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:40:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:10 smithi123 ceph-mon[24842]: Reconfiguring daemon mgr.smithi123.psjrfs on smithi123 2024-04-12T21:40:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:10 smithi123 ceph-mon[24842]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:11.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:11 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T21:40:11.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:11 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:11.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:11 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T21:40:11.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:11 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:12.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:12 smithi177 ceph-mon[26903]: Deploying daemon osd.0 on smithi123 2024-04-12T21:40:12.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:12 smithi177 ceph-mon[26903]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:12.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:12 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:12.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:12 smithi123 ceph-mon[24842]: Deploying daemon osd.0 on smithi123 2024-04-12T21:40:12.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:12 smithi123 ceph-mon[24842]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:12.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:12 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:13.558 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:13 smithi123 ceph-mon[24842]: Reconfiguring alertmanager.smithi123 (dependencies changed)... 2024-04-12T21:40:13.559 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:13 smithi123 ceph-mon[24842]: Reconfiguring daemon alertmanager.smithi123 on smithi123 2024-04-12T21:40:13.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:13 smithi177 ceph-mon[26903]: Reconfiguring alertmanager.smithi123 (dependencies changed)... 2024-04-12T21:40:13.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:13 smithi177 ceph-mon[26903]: Reconfiguring daemon alertmanager.smithi123 on smithi123 2024-04-12T21:40:14.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:14 smithi123 ceph-mon[24842]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:14.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:14 smithi177 ceph-mon[26903]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:16.609 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:16 smithi123 ceph-mon[24842]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:17.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:16 smithi177 ceph-mon[26903]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:18.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:18.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:18 smithi177 ceph-mon[26903]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:18.298 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:18.298 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:18 smithi123 ceph-mon[24842]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:20.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:20 smithi123 ceph-mon[24842]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:20.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:20 smithi123 ceph-mon[24842]: from='osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T21:40:20.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:20 smithi177 ceph-mon[26903]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:20.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:20 smithi177 ceph-mon[26903]: from='osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T21:40:21.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:21 smithi123 ceph-mon[24842]: from='osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T21:40:21.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:21 smithi123 ceph-mon[24842]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T21:40:21.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:21 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:21.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:21 smithi123 ceph-mon[24842]: from='osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:40:21.674 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:21 smithi177 ceph-mon[26903]: from='osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T21:40:21.674 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:21 smithi177 ceph-mon[26903]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T21:40:21.674 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:21 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:21.674 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:21 smithi177 ceph-mon[26903]: from='osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:40:22.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:22 smithi177 ceph-mon[26903]: from='osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-04-12T21:40:22.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:22 smithi177 ceph-mon[26903]: osdmap e8: 1 total, 0 up, 1 in 2024-04-12T21:40:22.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:22.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:22.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:22 smithi177 ceph-mon[26903]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:22.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:22.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:40:22.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:22.540 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:22 smithi123 ceph-mon[24842]: from='osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-04-12T21:40:22.540 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:22 smithi123 ceph-mon[24842]: osdmap e8: 1 total, 0 up, 1 in 2024-04-12T21:40:22.540 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:22.541 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:22.541 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:22 smithi123 ceph-mon[24842]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T21:40:22.541 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:22.541 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:40:22.541 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:23.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:23 smithi123 ceph-mon[24842]: purged_snaps scrub starts 2024-04-12T21:40:23.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:23 smithi123 ceph-mon[24842]: purged_snaps scrub ok 2024-04-12T21:40:23.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:23 smithi123 ceph-mon[24842]: Reconfiguring crash.smithi123 (monmap changed)... 2024-04-12T21:40:23.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:23 smithi123 ceph-mon[24842]: Reconfiguring daemon crash.smithi123 on smithi123 2024-04-12T21:40:23.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:23.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:23 smithi123 ceph-mon[24842]: osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118] boot 2024-04-12T21:40:23.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:23 smithi123 ceph-mon[24842]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T21:40:23.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:23.496 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 0 on host 'smithi123' 2024-04-12T21:40:23.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:23 smithi177 ceph-mon[26903]: purged_snaps scrub starts 2024-04-12T21:40:23.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:23 smithi177 ceph-mon[26903]: purged_snaps scrub ok 2024-04-12T21:40:23.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:23 smithi177 ceph-mon[26903]: Reconfiguring crash.smithi123 (monmap changed)... 2024-04-12T21:40:23.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:23 smithi177 ceph-mon[26903]: Reconfiguring daemon crash.smithi123 on smithi123 2024-04-12T21:40:23.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:23.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:23 smithi177 ceph-mon[26903]: osd.0 [v2:172.21.15.123:6802/1737101118,v1:172.21.15.123:6803/1737101118] boot 2024-04-12T21:40:23.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:23 smithi177 ceph-mon[26903]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T21:40:23.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:40:24.080 DEBUG:teuthology.orchestra.run.smithi123:osd.0> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.0.service 2024-04-12T21:40:24.083 INFO:tasks.cephadm:Deploying osd.1 on smithi123 with /dev/vg_nvme/lv_3... 2024-04-12T21:40:24.083 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-12T21:40:24.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:24 smithi123 ceph-mon[24842]: osdmap e10: 1 total, 1 up, 1 in 2024-04-12T21:40:24.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:24 smithi123 ceph-mon[24842]: pgmap v28: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:24.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:24.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:40:24.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:40:24.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:24 smithi177 ceph-mon[26903]: osdmap e10: 1 total, 1 up, 1 in 2024-04-12T21:40:24.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:24 smithi177 ceph-mon[26903]: pgmap v28: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:24.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:24.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:40:24.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:40:26.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:26 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:26.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:26 smithi177 ceph-mon[26903]: Reconfiguring grafana.smithi123 (dependencies changed)... 2024-04-12T21:40:26.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:26 smithi177 ceph-mon[26903]: Reconfiguring daemon grafana.smithi123 on smithi123 2024-04-12T21:40:26.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:26 smithi177 ceph-mon[26903]: pgmap v29: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:26.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:26 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:26.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:26 smithi123 ceph-mon[24842]: Reconfiguring grafana.smithi123 (dependencies changed)... 2024-04-12T21:40:26.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:26 smithi123 ceph-mon[24842]: Reconfiguring daemon grafana.smithi123 on smithi123 2024-04-12T21:40:26.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:26 smithi123 ceph-mon[24842]: pgmap v29: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:28.451 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-12T21:40:28.451 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-12T21:40:28.452 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2024-04-12T21:40:28.452 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2024-04-12T21:40:28.452 INFO:teuthology.orchestra.run.smithi123.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112489 s, 932 MB/s 2024-04-12T21:40:28.452 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2024-04-12T21:40:28.837 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:28 smithi123 ceph-mon[24842]: pgmap v30: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:29.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:28 smithi177 ceph-mon[26903]: pgmap v30: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:30.642 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:30 smithi123 ceph-mon[24842]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:30.670 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi123:vg_nvme/lv_3 2024-04-12T21:40:30.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:30 smithi177 ceph-mon[26903]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:32.384 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:32 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:32.385 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:32 smithi123 ceph-mon[24842]: Reconfiguring crash.smithi177 (monmap changed)... 2024-04-12T21:40:32.385 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:32 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:40:32.385 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:32 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:32.385 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:32 smithi123 ceph-mon[24842]: Reconfiguring daemon crash.smithi177 on smithi177 2024-04-12T21:40:32.385 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:32 smithi123 ceph-mon[24842]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:32.501 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:32 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:32.501 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:32 smithi177 ceph-mon[26903]: Reconfiguring crash.smithi177 (monmap changed)... 2024-04-12T21:40:32.502 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:32 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:40:32.502 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:32 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:32.502 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:32 smithi177 ceph-mon[26903]: Reconfiguring daemon crash.smithi177 on smithi177 2024-04-12T21:40:32.502 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:32 smithi177 ceph-mon[26903]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:33.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:33.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: Reconfiguring mgr.smithi177.whybza (monmap changed)... 2024-04-12T21:40:33.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:40:33.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:40:33.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:33.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: Reconfiguring daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:40:33.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:40:33.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:40:33.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:33 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:33.878 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:33.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: Reconfiguring mgr.smithi177.whybza (monmap changed)... 2024-04-12T21:40:33.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:40:33.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:40:33.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:33.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: Reconfiguring daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:40:33.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:40:33.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:40:33.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:33 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:34.641 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:34 smithi177 ceph-mon[26903]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:40:34.641 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:34 smithi177 ceph-mon[26903]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:34.641 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:34.641 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:40:34.641 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:40:34.641 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:34.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:34 smithi123 ceph-mon[24842]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:40:34.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:34 smithi123 ceph-mon[24842]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:34.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:34.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:40:34.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:40:34.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:35.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:35 smithi177 ceph-mon[26903]: Reconfiguring mon.smithi177 (monmap changed)... 2024-04-12T21:40:35.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:35 smithi177 ceph-mon[26903]: Reconfiguring daemon mon.smithi177 on smithi177 2024-04-12T21:40:35.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:35 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/4133362898' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f564e575-1d0c-4da6-bfdc-18c40fa666fb"}]: dispatch 2024-04-12T21:40:35.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:35 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/4133362898' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f564e575-1d0c-4da6-bfdc-18c40fa666fb"}]': finished 2024-04-12T21:40:35.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:35 smithi177 ceph-mon[26903]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T21:40:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:35 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:35 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:35 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:40:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:35 smithi123 ceph-mon[24842]: Reconfiguring mon.smithi177 (monmap changed)... 2024-04-12T21:40:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:35 smithi123 ceph-mon[24842]: Reconfiguring daemon mon.smithi177 on smithi177 2024-04-12T21:40:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:35 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/4133362898' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f564e575-1d0c-4da6-bfdc-18c40fa666fb"}]: dispatch 2024-04-12T21:40:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:35 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/4133362898' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f564e575-1d0c-4da6-bfdc-18c40fa666fb"}]': finished 2024-04-12T21:40:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:35 smithi123 ceph-mon[24842]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T21:40:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:35 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:35 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:35 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:40:36.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:36 smithi123 ceph-mon[24842]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:40:36.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:36 smithi123 ceph-mon[24842]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:36.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:36 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1727833012' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:40:36.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:36 smithi177 ceph-mon[26903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:40:36.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:36 smithi177 ceph-mon[26903]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:36.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:36 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1727833012' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:40:38.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:38 smithi177 ceph-mon[26903]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:38.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:38 smithi123 ceph-mon[24842]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:40.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:40 smithi123 ceph-mon[24842]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:40.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:40 smithi177 ceph-mon[26903]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:41.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:41 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T21:40:41.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:41 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:41.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:41 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T21:40:41.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:41 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:42.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:42 smithi177 ceph-mon[26903]: Deploying daemon osd.1 on smithi123 2024-04-12T21:40:42.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:42 smithi177 ceph-mon[26903]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:42.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:42 smithi123 ceph-mon[24842]: Deploying daemon osd.1 on smithi123 2024-04-12T21:40:42.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:42 smithi123 ceph-mon[24842]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:44.575 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:44 smithi123 ceph-mon[24842]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:45.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:44 smithi177 ceph-mon[26903]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:46.793 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:46 smithi123 ceph-mon[24842]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:46.793 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:46 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:47.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:46 smithi177 ceph-mon[26903]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:47.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:46 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:48.804 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:48 smithi123 ceph-mon[24842]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:48.836 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 1 on host 'smithi123' 2024-04-12T21:40:48.924 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:48 smithi177 ceph-mon[26903]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:49.480 DEBUG:teuthology.orchestra.run.smithi123:osd.1> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.1.service 2024-04-12T21:40:49.483 INFO:tasks.cephadm:Deploying osd.2 on smithi123 with /dev/vg_nvme/lv_2... 2024-04-12T21:40:49.483 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-12T21:40:50.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:50.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.123:9093"}]: dispatch 2024-04-12T21:40:50.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.123:9093"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.123:3000"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.123:3000"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.123:9095"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.123:9095"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:40:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:50.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:40:50.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: from='osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T21:40:50.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:49 smithi123 ceph-mon[24842]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.123:9093"}]: dispatch 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.123:9093"}]: dispatch 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.123:3000"}]: dispatch 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.123:3000"}]: dispatch 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:40:50.256 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.123:9095"}]: dispatch 2024-04-12T21:40:50.257 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.123:9095"}]: dispatch 2024-04-12T21:40:50.257 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:50.257 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:40:50.257 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:50.257 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:40:50.257 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: from='osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T21:40:50.257 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:49 smithi177 ceph-mon[26903]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:51.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:50 smithi177 ceph-mon[26903]: from='osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T21:40:51.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:50 smithi177 ceph-mon[26903]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T21:40:51.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:50 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:51.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:50 smithi177 ceph-mon[26903]: from='osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:40:51.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:50 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:51.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:50 smithi123 ceph-mon[24842]: from='osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T21:40:51.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:50 smithi123 ceph-mon[24842]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T21:40:51.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:50 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:51.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:50 smithi123 ceph-mon[24842]: from='osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:40:51.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:50 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:51.168 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:40:50 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[38116]: 2024-04-12T21:40:50.879+0000 7fd700d3f700 -1 osd.1 0 waiting for initial osdmap 2024-04-12T21:40:51.168 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:40:50 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[38116]: 2024-04-12T21:40:50.882+0000 7fd6fb6d6700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:40:51.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:51 smithi123 ceph-mon[24842]: from='osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-04-12T21:40:51.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:51 smithi123 ceph-mon[24842]: osdmap e13: 2 total, 1 up, 2 in 2024-04-12T21:40:51.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:51 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:51.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:51 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:51.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:51 smithi123 ceph-mon[24842]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:52.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:51 smithi177 ceph-mon[26903]: from='osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-04-12T21:40:52.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:51 smithi177 ceph-mon[26903]: osdmap e13: 2 total, 1 up, 2 in 2024-04-12T21:40:52.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:51 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:52.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:51 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:52.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:51 smithi177 ceph-mon[26903]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T21:40:52.751 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-12T21:40:52.751 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-12T21:40:52.751 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2024-04-12T21:40:52.751 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2024-04-12T21:40:52.752 INFO:teuthology.orchestra.run.smithi123.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0128193 s, 818 MB/s 2024-04-12T21:40:52.752 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 2024-04-12T21:40:52.752 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2024-04-12T21:40:53.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:52 smithi123 ceph-mon[24842]: purged_snaps scrub starts 2024-04-12T21:40:53.157 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:52 smithi123 ceph-mon[24842]: purged_snaps scrub ok 2024-04-12T21:40:53.157 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:52 smithi123 ceph-mon[24842]: osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237] boot 2024-04-12T21:40:53.157 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:52 smithi123 ceph-mon[24842]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T21:40:53.157 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:52 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:53.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:52 smithi177 ceph-mon[26903]: purged_snaps scrub starts 2024-04-12T21:40:53.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:52 smithi177 ceph-mon[26903]: purged_snaps scrub ok 2024-04-12T21:40:53.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:52 smithi177 ceph-mon[26903]: osd.1 [v2:172.21.15.123:6810/1173353237,v1:172.21.15.123:6811/1173353237] boot 2024-04-12T21:40:53.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:52 smithi177 ceph-mon[26903]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T21:40:53.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:52 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:40:53.370 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi123:vg_nvme/lv_2 2024-04-12T21:40:54.033 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:53 smithi123 ceph-mon[24842]: osdmap e15: 2 total, 2 up, 2 in 2024-04-12T21:40:54.033 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:53 smithi123 ceph-mon[24842]: pgmap v48: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:40:54.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:53 smithi177 ceph-mon[26903]: osdmap e15: 2 total, 2 up, 2 in 2024-04-12T21:40:54.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:53 smithi177 ceph-mon[26903]: pgmap v48: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:40:56.642 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:56 smithi123 ceph-mon[24842]: pgmap v49: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:40:56.642 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:40:56.642 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:40:56.642 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:57.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:56 smithi177 ceph-mon[26903]: pgmap v49: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:40:57.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:40:57.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:40:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:40:57.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:57 smithi123 ceph-mon[24842]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:40:57.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:57 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:58.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:57 smithi177 ceph-mon[26903]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:40:58.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:57 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:40:58.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:58 smithi123 ceph-mon[24842]: pgmap v50: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:40:58.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:58 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1903028304' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "66b74ab5-7a4e-4f96-a1ac-b5e4bf678d53"}]: dispatch 2024-04-12T21:40:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:58 smithi177 ceph-mon[26903]: pgmap v50: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:40:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:58 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1903028304' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "66b74ab5-7a4e-4f96-a1ac-b5e4bf678d53"}]: dispatch 2024-04-12T21:41:00.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:59 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1903028304' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "66b74ab5-7a4e-4f96-a1ac-b5e4bf678d53"}]': finished 2024-04-12T21:41:00.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:59 smithi177 ceph-mon[26903]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T21:41:00.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:59 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:00.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:40:59 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/4076277334' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:41:00.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:59 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1903028304' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "66b74ab5-7a4e-4f96-a1ac-b5e4bf678d53"}]': finished 2024-04-12T21:41:00.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:59 smithi123 ceph-mon[24842]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T21:41:00.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:59 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:00.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:40:59 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/4076277334' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:41:00.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:00 smithi123 ceph-mon[24842]: pgmap v52: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:01.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:00 smithi177 ceph-mon[26903]: pgmap v52: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:02.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:02 smithi123 ceph-mon[24842]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:03.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:02 smithi177 ceph-mon[26903]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:04 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:05.119 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:05.119 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:05.119 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:05.119 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:05.120 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:05.120 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:05.120 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:05.120 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:05.120 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:04 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:05 smithi123 ceph-mon[24842]: Detected new or changed devices on smithi123 2024-04-12T21:41:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:05 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi123 to 6432M 2024-04-12T21:41:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:06.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:05 smithi177 ceph-mon[26903]: Detected new or changed devices on smithi123 2024-04-12T21:41:06.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:05 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi123 to 6432M 2024-04-12T21:41:06.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:06.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:06 smithi123 ceph-mon[24842]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:06.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T21:41:06.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:06 smithi177 ceph-mon[26903]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T21:41:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:07.907 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:07 smithi123 ceph-mon[24842]: Deploying daemon osd.2 on smithi123 2024-04-12T21:41:08.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:07 smithi177 ceph-mon[26903]: Deploying daemon osd.2 on smithi123 2024-04-12T21:41:08.871 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:08 smithi123 ceph-mon[24842]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:09.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:08 smithi177 ceph-mon[26903]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:10.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:10 smithi123 ceph-mon[24842]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:10.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:10 smithi177 ceph-mon[26903]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:12.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:12 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:12.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:12 smithi177 ceph-mon[26903]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:12.541 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:12 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:12.541 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:12 smithi123 ceph-mon[24842]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:14.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:14 smithi123 ceph-mon[24842]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:14.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:14 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:15.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:14 smithi177 ceph-mon[26903]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:15.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:14 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:16.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:16 smithi123 ceph-mon[24842]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:16.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:16.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:16.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:16.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:16.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:16 smithi177 ceph-mon[26903]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:17.492 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 2 on host 'smithi123' 2024-04-12T21:41:17.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:17 smithi177 ceph-mon[26903]: from='osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T21:41:17.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:17 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:17.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:17 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:17.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:17 smithi123 ceph-mon[24842]: from='osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T21:41:17.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:17 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:17.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:17 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:18.112 DEBUG:teuthology.orchestra.run.smithi123:osd.2> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.2.service 2024-04-12T21:41:18.115 INFO:tasks.cephadm:Deploying osd.3 on smithi123 with /dev/vg_nvme/lv_1... 2024-04-12T21:41:18.115 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-12T21:41:18.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:18 smithi177 ceph-mon[26903]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:18.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:18 smithi177 ceph-mon[26903]: from='osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T21:41:18.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:18 smithi177 ceph-mon[26903]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T21:41:18.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:18.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:18 smithi177 ceph-mon[26903]: from='osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:41:18.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:18 smithi123 ceph-mon[24842]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:18.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:18 smithi123 ceph-mon[24842]: from='osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T21:41:18.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:18 smithi123 ceph-mon[24842]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T21:41:18.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:18.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:18 smithi123 ceph-mon[24842]: from='osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:41:18.803 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:41:18 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[41974]: 2024-04-12T21:41:18.521+0000 7fd3a7545700 -1 osd.2 0 waiting for initial osdmap 2024-04-12T21:41:18.803 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:41:18 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[41974]: 2024-04-12T21:41:18.524+0000 7fd3a26dd700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:41:19.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:19 smithi177 ceph-mon[26903]: from='osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-04-12T21:41:19.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:19 smithi177 ceph-mon[26903]: osdmap e18: 3 total, 2 up, 3 in 2024-04-12T21:41:19.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:19 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:19.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:19 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:19.808 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:19 smithi123 ceph-mon[24842]: from='osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-04-12T21:41:19.808 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:19 smithi123 ceph-mon[24842]: osdmap e18: 3 total, 2 up, 3 in 2024-04-12T21:41:19.808 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:19 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:19.808 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:19 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:20.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:20 smithi123 ceph-mon[24842]: purged_snaps scrub starts 2024-04-12T21:41:20.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:20 smithi123 ceph-mon[24842]: purged_snaps scrub ok 2024-04-12T21:41:20.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:20 smithi123 ceph-mon[24842]: pgmap v64: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:20.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:20 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:20.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:20 smithi123 ceph-mon[24842]: osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435] boot 2024-04-12T21:41:20.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:20 smithi123 ceph-mon[24842]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T21:41:20.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:20 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:20.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:20 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-12T21:41:20.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:20 smithi177 ceph-mon[26903]: purged_snaps scrub starts 2024-04-12T21:41:20.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:20 smithi177 ceph-mon[26903]: purged_snaps scrub ok 2024-04-12T21:41:20.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:20 smithi177 ceph-mon[26903]: pgmap v64: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-12T21:41:20.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:20 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:20.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:20 smithi177 ceph-mon[26903]: osd.2 [v2:172.21.15.123:6818/1826006435,v1:172.21.15.123:6819/1826006435] boot 2024-04-12T21:41:20.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:20 smithi177 ceph-mon[26903]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T21:41:20.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:20 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:41:20.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:20 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-12T21:41:21.041 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-12T21:41:21.041 INFO:teuthology.orchestra.run.smithi123.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-12T21:41:21.041 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10+0 records in 2024-04-12T21:41:21.041 INFO:teuthology.orchestra.run.smithi123.stderr:10+0 records out 2024-04-12T21:41:21.041 INFO:teuthology.orchestra.run.smithi123.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125176 s, 838 MB/s 2024-04-12T21:41:21.041 INFO:teuthology.orchestra.run.smithi123.stderr:--> Zapping successful for: 2024-04-12T21:41:21.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:21 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-12T21:41:21.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:21 smithi123 ceph-mon[24842]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T21:41:21.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:21 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T21:41:22.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:21 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-12T21:41:22.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:21 smithi177 ceph-mon[26903]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T21:41:22.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:21 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T21:41:22.530 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43276]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T21:41:22.531 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43276]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:22.531 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43276]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:22.739 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi123:vg_nvme/lv_1 2024-04-12T21:41:22.800 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:22 smithi123 ceph-mon[24842]: pgmap v67: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:22.800 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T21:41:22.800 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:22 smithi123 ceph-mon[24842]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T21:41:22.800 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43311]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T21:41:22.800 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43311]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:22.800 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43311]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:23.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:22 smithi177 ceph-mon[26903]: pgmap v67: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:23.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T21:41:23.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:22 smithi177 ceph-mon[26903]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T21:41:23.068 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43332]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T21:41:23.068 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43332]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:23.068 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:41:22 smithi123 sudo[43332]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:23.328 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43388]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T21:41:23.328 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43388]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:23.328 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43388]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:23.328 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43393]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T21:41:23.328 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43393]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:23.328 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43393]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:23.581 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:23 smithi123 ceph-mon[24842]: osdmap e22: 3 total, 3 up, 3 in 2024-04-12T21:41:23.581 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:23.582 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:41:23.582 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:41:23.833 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43446]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T21:41:23.833 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43446]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:23.833 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43446]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:24.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:23 smithi177 ceph-mon[26903]: osdmap e22: 3 total, 3 up, 3 in 2024-04-12T21:41:24.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:24.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:41:24.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:41:24.089 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43454]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T21:41:24.089 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43454]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:24.089 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:23 smithi123 sudo[43454]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:24.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 sudo[43470]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T21:41:24.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 sudo[43470]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:24.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 sudo[43470]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:24.581 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 sudo[29109]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T21:41:24.582 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 sudo[29109]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:24.582 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 sudo[29109]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:24.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 ceph-mon[24842]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:24.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 ceph-mon[24842]: mgrmap e19: smithi123.psjrfs(active, since 2m), standbys: smithi177.whybza 2024-04-12T21:41:24.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 ceph-mon[24842]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T21:41:24.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 ceph-mon[24842]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T21:41:24.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:41:24.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:41:24.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:41:24.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:41:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 sudo[29112]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T21:41:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 sudo[29112]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T21:41:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 sudo[29112]: pam_unix(sudo:session): session closed for user root 2024-04-12T21:41:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 ceph-mon[26903]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 ceph-mon[26903]: mgrmap e19: smithi123.psjrfs(active, since 2m), standbys: smithi177.whybza 2024-04-12T21:41:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 ceph-mon[26903]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T21:41:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 ceph-mon[26903]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T21:41:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:41:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:41:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:41:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:41:25.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:25 smithi123 ceph-mon[24842]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T21:41:25.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:25 smithi123 ceph-mon[24842]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T21:41:25.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:25 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:25.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:25 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:25.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:25 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:41:25.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:25 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:41:25.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:25 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:25 smithi177 ceph-mon[26903]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T21:41:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:25 smithi177 ceph-mon[26903]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T21:41:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:25 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:25 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:25 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:41:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:25 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:41:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:25 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:26.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:26 smithi123 ceph-mon[24842]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:41:26.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:26 smithi123 ceph-mon[24842]: pgmap v71: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:27.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:26 smithi177 ceph-mon[26903]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:41:27.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:26 smithi177 ceph-mon[26903]: pgmap v71: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:28.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:28 smithi123 ceph-mon[24842]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:28.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:28 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2652901792' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25ce9222-04fa-4b49-9312-4567174dc425"}]: dispatch 2024-04-12T21:41:28.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:28 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2652901792' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "25ce9222-04fa-4b49-9312-4567174dc425"}]': finished 2024-04-12T21:41:28.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:28 smithi123 ceph-mon[24842]: osdmap e23: 4 total, 3 up, 4 in 2024-04-12T21:41:28.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:28 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:29.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:28 smithi177 ceph-mon[26903]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:29.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:28 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2652901792' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "25ce9222-04fa-4b49-9312-4567174dc425"}]: dispatch 2024-04-12T21:41:29.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:28 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2652901792' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "25ce9222-04fa-4b49-9312-4567174dc425"}]': finished 2024-04-12T21:41:29.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:28 smithi177 ceph-mon[26903]: osdmap e23: 4 total, 3 up, 4 in 2024-04-12T21:41:29.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:28 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:29.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:29 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1907024209' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:41:30.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:29 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1907024209' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:41:30.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:30 smithi123 ceph-mon[24842]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-12T21:41:31.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:30 smithi177 ceph-mon[26903]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-12T21:41:32.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:32 smithi123 ceph-mon[24842]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-12T21:41:33.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:32 smithi177 ceph-mon[26903]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-12T21:41:34.856 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T21:41:34.856 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:34.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:34.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:34.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:34.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:34.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:34.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:34.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:34.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:35.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T21:41:35.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:35.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:35.864 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:35 smithi123 ceph-mon[24842]: Detected new or changed devices on smithi123 2024-04-12T21:41:35.864 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:35 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi123 to 4288M 2024-04-12T21:41:36.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:35 smithi177 ceph-mon[26903]: Detected new or changed devices on smithi123 2024-04-12T21:41:36.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:35 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi123 to 4288M 2024-04-12T21:41:36.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:36 smithi123 ceph-mon[24842]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T21:41:36.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:36 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T21:41:36.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:36 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:37.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:36 smithi177 ceph-mon[26903]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T21:41:37.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:36 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T21:41:37.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:36 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:37.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:37 smithi123 ceph-mon[24842]: Deploying daemon osd.3 on smithi123 2024-04-12T21:41:38.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:37 smithi177 ceph-mon[26903]: Deploying daemon osd.3 on smithi123 2024-04-12T21:41:38.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:38 smithi123 ceph-mon[24842]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T21:41:39.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:38 smithi177 ceph-mon[26903]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T21:41:40.552 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:40 smithi123 ceph-mon[24842]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:40.552 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:40 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:40.552 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:40 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:40.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:40 smithi177 ceph-mon[26903]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:40.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:40 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:40.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:40 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:42.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:42 smithi177 ceph-mon[26903]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:42.862 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:42 smithi123 ceph-mon[24842]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:44.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:44.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:44.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:44.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:44 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:44.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:44 smithi177 ceph-mon[26903]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:44.584 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:44.584 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:44.584 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:44.584 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:44 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:44.584 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:44 smithi123 ceph-mon[24842]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:46.463 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 3 on host 'smithi123' 2024-04-12T21:41:46.806 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:46 smithi123 ceph-mon[24842]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:46.806 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:46 smithi123 ceph-mon[24842]: from='osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T21:41:46.806 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:46 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:47.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:46 smithi177 ceph-mon[26903]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:47.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:46 smithi177 ceph-mon[26903]: from='osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T21:41:47.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:46 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:47.651 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:47 smithi123 ceph-mon[24842]: from='osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T21:41:47.651 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:47 smithi123 ceph-mon[24842]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T21:41:47.651 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:47 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:47.651 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:47 smithi123 ceph-mon[24842]: from='osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:41:47.674 DEBUG:teuthology.orchestra.run.smithi123:osd.3> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.3.service 2024-04-12T21:41:47.676 INFO:tasks.cephadm:Deploying osd.4 on smithi177 with /dev/vg_nvme/lv_4... 2024-04-12T21:41:47.676 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-12T21:41:47.995 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:47 smithi177 ceph-mon[26903]: from='osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T21:41:47.995 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:47 smithi177 ceph-mon[26903]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T21:41:47.995 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:47 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:47.995 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:47 smithi177 ceph-mon[26903]: from='osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:41:48.900 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:48 smithi177 ceph-mon[26903]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:48.901 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:48 smithi177 ceph-mon[26903]: from='osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-04-12T21:41:48.901 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:48 smithi177 ceph-mon[26903]: osdmap e25: 4 total, 3 up, 4 in 2024-04-12T21:41:48.901 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:48 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:48.901 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:48 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:48 smithi123 ceph-mon[24842]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:48 smithi123 ceph-mon[24842]: from='osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-04-12T21:41:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:48 smithi123 ceph-mon[24842]: osdmap e25: 4 total, 3 up, 4 in 2024-04-12T21:41:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:48 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:48 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:49.490 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-12T21:41:49.490 INFO:teuthology.orchestra.run.smithi177.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-12T21:41:49.490 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10+0 records in 2024-04-12T21:41:49.491 INFO:teuthology.orchestra.run.smithi177.stderr:10+0 records out 2024-04-12T21:41:49.491 INFO:teuthology.orchestra.run.smithi177.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111819 s, 938 MB/s 2024-04-12T21:41:49.491 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 2024-04-12T21:41:49.491 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping successful for: 2024-04-12T21:41:49.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:49 smithi123 ceph-mon[24842]: purged_snaps scrub starts 2024-04-12T21:41:49.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:49 smithi123 ceph-mon[24842]: purged_snaps scrub ok 2024-04-12T21:41:49.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:49.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:49 smithi123 ceph-mon[24842]: osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540] boot 2024-04-12T21:41:49.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:49 smithi123 ceph-mon[24842]: osdmap e26: 4 total, 4 up, 4 in 2024-04-12T21:41:49.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:49 smithi177 ceph-mon[26903]: purged_snaps scrub starts 2024-04-12T21:41:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:49 smithi177 ceph-mon[26903]: purged_snaps scrub ok 2024-04-12T21:41:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:49 smithi177 ceph-mon[26903]: osd.3 [v2:172.21.15.123:6826/270637540,v1:172.21.15.123:6827/270637540] boot 2024-04-12T21:41:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:49 smithi177 ceph-mon[26903]: osdmap e26: 4 total, 4 up, 4 in 2024-04-12T21:41:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:41:50.042 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi177:vg_nvme/lv_4 2024-04-12T21:41:50.600 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:50 smithi177 ceph-mon[26903]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:50.600 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:50 smithi177 ceph-mon[26903]: osdmap e27: 4 total, 4 up, 4 in 2024-04-12T21:41:50.600 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:50 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:50.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:50 smithi123 ceph-mon[24842]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T21:41:50.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:50 smithi123 ceph-mon[24842]: osdmap e27: 4 total, 4 up, 4 in 2024-04-12T21:41:50.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:50 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:52 smithi123 ceph-mon[24842]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:41:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:52 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:41:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:52 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:41:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:52 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:52.975 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:52 smithi177 ceph-mon[26903]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:41:52.976 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:52 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:41:52.976 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:52 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:41:52.976 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:52 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:53.803 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:53.804 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:53 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:53.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:41:53.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:53.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:53.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:53.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:53.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:41:53.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:53.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:53.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:41:53.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:41:53.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:53 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:41:54.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:54 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi123 to 3216M 2024-04-12T21:41:54.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:54 smithi123 ceph-mon[24842]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:41:54.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:54 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/1875128846' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a"}]: dispatch 2024-04-12T21:41:54.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:54 smithi123 ceph-mon[24842]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a"}]: dispatch 2024-04-12T21:41:54.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:54 smithi123 ceph-mon[24842]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a"}]': finished 2024-04-12T21:41:54.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:54 smithi123 ceph-mon[24842]: osdmap e28: 5 total, 4 up, 5 in 2024-04-12T21:41:54.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:54 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:41:55.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:54 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi123 to 3216M 2024-04-12T21:41:55.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:54 smithi177 ceph-mon[26903]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:41:55.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:54 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/1875128846' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a"}]: dispatch 2024-04-12T21:41:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:54 smithi177 ceph-mon[26903]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a"}]: dispatch 2024-04-12T21:41:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:54 smithi177 ceph-mon[26903]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a"}]': finished 2024-04-12T21:41:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:54 smithi177 ceph-mon[26903]: osdmap e28: 5 total, 4 up, 5 in 2024-04-12T21:41:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:54 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:41:55.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:55 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/4267103658' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:41:56.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:55 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/4267103658' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:41:56.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:56 smithi123 ceph-mon[24842]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:41:57.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:56 smithi177 ceph-mon[26903]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:41:58.249 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:58 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:58.249 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:58 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:58.249 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:41:58 smithi177 ceph-mon[26903]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:41:58.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:58 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:58.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:58 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:41:58.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:41:58 smithi123 ceph-mon[24842]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:00.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:00 smithi123 ceph-mon[24842]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:00.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:00 smithi177 ceph-mon[26903]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:01 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T21:42:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:01 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:01 smithi123 ceph-mon[24842]: Deploying daemon osd.4 on smithi177 2024-04-12T21:42:01.742 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:01 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T21:42:01.742 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:01 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:01.743 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:01 smithi177 ceph-mon[26903]: Deploying daemon osd.4 on smithi177 2024-04-12T21:42:02.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:02 smithi123 ceph-mon[24842]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:02.716 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:02 smithi177 ceph-mon[26903]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:04.771 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:04 smithi177 ceph-mon[26903]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:04.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:04 smithi123 ceph-mon[24842]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:06.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:06.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:06.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:06.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:06.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:06 smithi123 ceph-mon[24842]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:06.499 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:06.500 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:06.500 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:06.500 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:06.500 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:06 smithi177 ceph-mon[26903]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:08.829 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:08 smithi177 ceph-mon[26903]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:08.829 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:08 smithi177 ceph-mon[26903]: from='osd.4 [v2:172.21.15.177:6800/2901225367,v1:172.21.15.177:6801/2901225367]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T21:42:08.829 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:08 smithi177 ceph-mon[26903]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T21:42:08.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:08 smithi123 ceph-mon[24842]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:08.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:08 smithi123 ceph-mon[24842]: from='osd.4 [v2:172.21.15.177:6800/2901225367,v1:172.21.15.177:6801/2901225367]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T21:42:08.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:08 smithi123 ceph-mon[24842]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T21:42:09.136 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 4 on host 'smithi177' 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='osd.4 [v2:172.21.15.177:6800/2901225367,v1:172.21.15.177:6801/2901225367]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:09.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:09 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:09.788 DEBUG:teuthology.orchestra.run.smithi177:osd.4> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.4.service 2024-04-12T21:42:09.790 INFO:tasks.cephadm:Deploying osd.5 on smithi177 with /dev/vg_nvme/lv_3... 2024-04-12T21:42:09.790 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-12T21:42:09.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T21:42:09.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T21:42:09.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:42:09.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='osd.4 [v2:172.21.15.177:6800/2901225367,v1:172.21.15.177:6801/2901225367]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:09.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:09.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:09.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:09.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:09.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:09.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:09.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:09 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:10.856 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:10 smithi177 ceph-mon[26903]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:10.856 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:10 smithi177 ceph-mon[26903]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-04-12T21:42:10.856 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:10 smithi177 ceph-mon[26903]: osdmap e30: 5 total, 4 up, 5 in 2024-04-12T21:42:10.856 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:42:10.856 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:42:10.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:10 smithi123 ceph-mon[24842]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:10.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:10 smithi123 ceph-mon[24842]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-04-12T21:42:10.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:10 smithi123 ceph-mon[24842]: osdmap e30: 5 total, 4 up, 5 in 2024-04-12T21:42:10.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:42:10.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:42:11.800 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:11 smithi177 ceph-mon[26903]: purged_snaps scrub starts 2024-04-12T21:42:11.800 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:11 smithi177 ceph-mon[26903]: purged_snaps scrub ok 2024-04-12T21:42:11.800 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:11 smithi177 ceph-mon[26903]: osd.4 [v2:172.21.15.177:6800/2901225367,v1:172.21.15.177:6801/2901225367] boot 2024-04-12T21:42:11.800 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:11 smithi177 ceph-mon[26903]: osdmap e31: 5 total, 5 up, 5 in 2024-04-12T21:42:11.800 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:11 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:42:11.800 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:11 smithi177 ceph-mon[26903]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T21:42:11.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:11 smithi123 ceph-mon[24842]: purged_snaps scrub starts 2024-04-12T21:42:11.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:11 smithi123 ceph-mon[24842]: purged_snaps scrub ok 2024-04-12T21:42:11.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:11 smithi123 ceph-mon[24842]: osd.4 [v2:172.21.15.177:6800/2901225367,v1:172.21.15.177:6801/2901225367] boot 2024-04-12T21:42:11.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:11 smithi123 ceph-mon[24842]: osdmap e31: 5 total, 5 up, 5 in 2024-04-12T21:42:11.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:11 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:42:11.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:11 smithi123 ceph-mon[24842]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T21:42:12.635 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:12 smithi177 ceph-mon[26903]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:12.635 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:12 smithi177 ceph-mon[26903]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T21:42:12.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:12 smithi123 ceph-mon[24842]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T21:42:12.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:12 smithi123 ceph-mon[24842]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T21:42:13.403 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-12T21:42:13.403 INFO:teuthology.orchestra.run.smithi177.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-12T21:42:13.403 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10+0 records in 2024-04-12T21:42:13.403 INFO:teuthology.orchestra.run.smithi177.stderr:10+0 records out 2024-04-12T21:42:13.403 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110528 s, 949 MB/s 2024-04-12T21:42:13.403 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping successful for: 2024-04-12T21:42:14.241 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi177:vg_nvme/lv_3 2024-04-12T21:42:14.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:14 smithi177 ceph-mon[26903]: pgmap v106: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-12T21:42:14.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:14 smithi177 ceph-mon[26903]: Detected new or changed devices on smithi177 2024-04-12T21:42:14.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:14 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:14.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:14 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:42:14.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:14 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi177 to 15936M 2024-04-12T21:42:14.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:14 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:14.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:14 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:14.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:14 smithi177 ceph-mon[26903]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T21:42:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:14 smithi123 ceph-mon[24842]: pgmap v106: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-12T21:42:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:14 smithi123 ceph-mon[24842]: Detected new or changed devices on smithi177 2024-04-12T21:42:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:14 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:14 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:42:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:14 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi177 to 15936M 2024-04-12T21:42:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:14 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:14 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:14 smithi123 ceph-mon[24842]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T21:42:15.736 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:15 smithi177 ceph-mon[26903]: osdmap e35: 5 total, 5 up, 5 in 2024-04-12T21:42:15.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:15 smithi123 ceph-mon[24842]: osdmap e35: 5 total, 5 up, 5 in 2024-04-12T21:42:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:16 smithi177 ceph-mon[26903]: pgmap v109: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-12T21:42:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:42:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:42:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:16.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:16 smithi123 ceph-mon[24842]: pgmap v109: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-12T21:42:16.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:42:16.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:42:16.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:17.738 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:17 smithi177 ceph-mon[26903]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:42:17.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:17 smithi123 ceph-mon[24842]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:42:18.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:18 smithi123 ceph-mon[24842]: pgmap v110: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-12T21:42:18.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:18 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/2134957991' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "77011f8f-279b-4bd1-a254-3d58ddaa0bbf"}]: dispatch 2024-04-12T21:42:18.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:18 smithi123 ceph-mon[24842]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "77011f8f-279b-4bd1-a254-3d58ddaa0bbf"}]: dispatch 2024-04-12T21:42:18.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:18 smithi123 ceph-mon[24842]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "77011f8f-279b-4bd1-a254-3d58ddaa0bbf"}]': finished 2024-04-12T21:42:18.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:18 smithi123 ceph-mon[24842]: osdmap e36: 6 total, 5 up, 6 in 2024-04-12T21:42:18.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:19.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:18 smithi177 ceph-mon[26903]: pgmap v110: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-12T21:42:19.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:18 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/2134957991' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "77011f8f-279b-4bd1-a254-3d58ddaa0bbf"}]: dispatch 2024-04-12T21:42:19.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:18 smithi177 ceph-mon[26903]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "77011f8f-279b-4bd1-a254-3d58ddaa0bbf"}]: dispatch 2024-04-12T21:42:19.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:18 smithi177 ceph-mon[26903]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "77011f8f-279b-4bd1-a254-3d58ddaa0bbf"}]': finished 2024-04-12T21:42:19.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:18 smithi177 ceph-mon[26903]: osdmap e36: 6 total, 5 up, 6 in 2024-04-12T21:42:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:19.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:19 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/3518082219' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:42:20.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:19 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/3518082219' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:42:20.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:20 smithi123 ceph-mon[24842]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:21.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:20 smithi177 ceph-mon[26903]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:22.745 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:22 smithi177 ceph-mon[26903]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:22.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:22 smithi123 ceph-mon[24842]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:23.657 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:42:23.657 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:42:23.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:42:23.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:42:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:24 smithi123 ceph-mon[24842]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:24 smithi177 ceph-mon[26903]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:25.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:25 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T21:42:25.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:25 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:25.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:25 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T21:42:25.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:25 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:26.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:26 smithi123 ceph-mon[24842]: Deploying daemon osd.5 on smithi177 2024-04-12T21:42:26.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:26 smithi123 ceph-mon[24842]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:27.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:26 smithi177 ceph-mon[26903]: Deploying daemon osd.5 on smithi177 2024-04-12T21:42:27.017 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:26 smithi177 ceph-mon[26903]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:28.899 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:28 smithi177 ceph-mon[26903]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:28.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:28 smithi123 ceph-mon[24842]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:30.456 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:30 smithi177 ceph-mon[26903]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:30.456 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:30 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:30.456 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:30 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:30.456 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:30 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:30.456 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:30 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:30.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:30 smithi123 ceph-mon[24842]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:30.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:30.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:30.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:30.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:30 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:32.834 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:32 smithi177 ceph-mon[26903]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:32.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:32 smithi123 ceph-mon[24842]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:33.041 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 5 on host 'smithi177' 2024-04-12T21:42:33.647 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:33 smithi177 ceph-mon[26903]: from='osd.5 [v2:172.21.15.177:6808/3827733016,v1:172.21.15.177:6809/3827733016]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T21:42:33.647 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:33 smithi177 ceph-mon[26903]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T21:42:33.647 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:33 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:33.671 DEBUG:teuthology.orchestra.run.smithi177:osd.5> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.5.service 2024-04-12T21:42:33.673 INFO:tasks.cephadm:Deploying osd.6 on smithi177 with /dev/vg_nvme/lv_2... 2024-04-12T21:42:33.673 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-12T21:42:33.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:33 smithi123 ceph-mon[24842]: from='osd.5 [v2:172.21.15.177:6808/3827733016,v1:172.21.15.177:6809/3827733016]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T21:42:33.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:33 smithi123 ceph-mon[24842]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T21:42:33.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:33 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:34.820 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:34 smithi177 ceph-mon[26903]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:34.820 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:34 smithi177 ceph-mon[26903]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T21:42:34.820 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:34 smithi177 ceph-mon[26903]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T21:42:34.820 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:34 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:34.820 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:34 smithi177 ceph-mon[26903]: from='osd.5 [v2:172.21.15.177:6808/3827733016,v1:172.21.15.177:6809/3827733016]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:34.820 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:34 smithi177 ceph-mon[26903]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:34.820 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:42:34 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[34005]: 2024-04-12T21:42:34.572+0000 7f3e6f1b6700 -1 osd.5 0 waiting for initial osdmap 2024-04-12T21:42:34.820 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:42:34 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[34005]: 2024-04-12T21:42:34.577+0000 7f3e66346700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:42:34.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:34 smithi123 ceph-mon[24842]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:34.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:34 smithi123 ceph-mon[24842]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T21:42:34.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:34 smithi123 ceph-mon[24842]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T21:42:34.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:34 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:34.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:34 smithi123 ceph-mon[24842]: from='osd.5 [v2:172.21.15.177:6808/3827733016,v1:172.21.15.177:6809/3827733016]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:34.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:34 smithi123 ceph-mon[24842]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:35.722 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:35 smithi177 ceph-mon[26903]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-04-12T21:42:35.722 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:35 smithi177 ceph-mon[26903]: osdmap e38: 6 total, 5 up, 6 in 2024-04-12T21:42:35.722 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:35 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:35.722 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:35 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:35 smithi123 ceph-mon[24842]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-04-12T21:42:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:35 smithi123 ceph-mon[24842]: osdmap e38: 6 total, 5 up, 6 in 2024-04-12T21:42:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:35 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:35 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:36.225 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-12T21:42:36.226 INFO:teuthology.orchestra.run.smithi177.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-12T21:42:36.226 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10+0 records in 2024-04-12T21:42:36.226 INFO:teuthology.orchestra.run.smithi177.stderr:10+0 records out 2024-04-12T21:42:36.226 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012058 s, 870 MB/s 2024-04-12T21:42:36.226 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping successful for: 2024-04-12T21:42:36.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:36 smithi123 ceph-mon[24842]: purged_snaps scrub starts 2024-04-12T21:42:36.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:36 smithi123 ceph-mon[24842]: purged_snaps scrub ok 2024-04-12T21:42:36.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:36 smithi123 ceph-mon[24842]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:36.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:36 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:36.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:36 smithi123 ceph-mon[24842]: osd.5 [v2:172.21.15.177:6808/3827733016,v1:172.21.15.177:6809/3827733016] boot 2024-04-12T21:42:36.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:36 smithi123 ceph-mon[24842]: osdmap e39: 6 total, 6 up, 6 in 2024-04-12T21:42:36.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:36 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:36.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:36 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:37.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:36 smithi177 ceph-mon[26903]: purged_snaps scrub starts 2024-04-12T21:42:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:36 smithi177 ceph-mon[26903]: purged_snaps scrub ok 2024-04-12T21:42:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:36 smithi177 ceph-mon[26903]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T21:42:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:36 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:36 smithi177 ceph-mon[26903]: osd.5 [v2:172.21.15.177:6808/3827733016,v1:172.21.15.177:6809/3827733016] boot 2024-04-12T21:42:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:36 smithi177 ceph-mon[26903]: osdmap e39: 6 total, 6 up, 6 in 2024-04-12T21:42:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:36 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:42:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:36 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:37.289 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi177:vg_nvme/lv_2 2024-04-12T21:42:37.752 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:37 smithi177 ceph-mon[26903]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T21:42:37.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:37 smithi123 ceph-mon[24842]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T21:42:38.717 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:38 smithi177 ceph-mon[26903]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:38.718 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:38 smithi177 ceph-mon[26903]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T21:42:38.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:38 smithi123 ceph-mon[24842]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:38.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:38 smithi123 ceph-mon[24842]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T21:42:39.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:39 smithi177 ceph-mon[26903]: osdmap e43: 6 total, 6 up, 6 in 2024-04-12T21:42:39.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T21:42:39.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:39.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:42:39.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:42:39.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:39 smithi123 ceph-mon[24842]: osdmap e43: 6 total, 6 up, 6 in 2024-04-12T21:42:40.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:40 smithi123 ceph-mon[24842]: pgmap v128: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:40.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:40 smithi123 ceph-mon[24842]: Detected new or changed devices on smithi177 2024-04-12T21:42:40.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:40 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi177 to 7968M 2024-04-12T21:42:40.999 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:40 smithi177 ceph-mon[26903]: pgmap v128: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:41.000 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:40 smithi177 ceph-mon[26903]: Detected new or changed devices on smithi177 2024-04-12T21:42:41.000 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:40 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi177 to 7968M 2024-04-12T21:42:41.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:41 smithi123 ceph-mon[24842]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:42:41.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:41 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:42:41.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:41 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:42:41.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:41 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:42.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:41 smithi177 ceph-mon[26903]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:42:42.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:41 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:42:42.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:41 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:42:42.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:41 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:42.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:42 smithi177 ceph-mon[26903]: pgmap v130: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:42.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:42 smithi123 ceph-mon[24842]: pgmap v130: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:43.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:43 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/169331555' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f0b81d8-73ce-4aef-95d1-c6216242457e"}]: dispatch 2024-04-12T21:42:43.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:43 smithi123 ceph-mon[24842]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f0b81d8-73ce-4aef-95d1-c6216242457e"}]: dispatch 2024-04-12T21:42:43.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:43 smithi123 ceph-mon[24842]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f0b81d8-73ce-4aef-95d1-c6216242457e"}]': finished 2024-04-12T21:42:43.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:43 smithi123 ceph-mon[24842]: osdmap e44: 7 total, 6 up, 7 in 2024-04-12T21:42:43.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:43 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:42:44.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:43 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/169331555' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f0b81d8-73ce-4aef-95d1-c6216242457e"}]: dispatch 2024-04-12T21:42:44.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:43 smithi177 ceph-mon[26903]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f0b81d8-73ce-4aef-95d1-c6216242457e"}]: dispatch 2024-04-12T21:42:44.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:43 smithi177 ceph-mon[26903]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f0b81d8-73ce-4aef-95d1-c6216242457e"}]': finished 2024-04-12T21:42:44.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:43 smithi177 ceph-mon[26903]: osdmap e44: 7 total, 6 up, 7 in 2024-04-12T21:42:44.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:43 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:42:44.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:44 smithi123 ceph-mon[24842]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:44.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:44 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/791984418' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:42:45.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:44 smithi177 ceph-mon[26903]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:45.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:44 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/791984418' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:42:46.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:46 smithi177 ceph-mon[26903]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:46.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:46 smithi123 ceph-mon[24842]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:48 smithi123 ceph-mon[24842]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:49.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:48 smithi177 ceph-mon[26903]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T21:42:49.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:49.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T21:42:49.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:50.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:50 smithi123 ceph-mon[24842]: Deploying daemon osd.6 on smithi177 2024-04-12T21:42:50.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:50 smithi123 ceph-mon[24842]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:50.929 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:50 smithi177 ceph-mon[26903]: Deploying daemon osd.6 on smithi177 2024-04-12T21:42:50.929 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:50 smithi177 ceph-mon[26903]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:52.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:52 smithi123 ceph-mon[24842]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:53.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:52 smithi177 ceph-mon[26903]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:54.772 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:54 smithi177 ceph-mon[26903]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:54.772 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:54 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:54.772 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:54 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:54.773 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:54 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:54.773 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:54 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:54.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:54 smithi123 ceph-mon[24842]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:54.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:54 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:54.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:54 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:42:54.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:54 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:42:54.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:54 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:42:56.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:56 smithi177 ceph-mon[26903]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:56.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:56 smithi123 ceph-mon[24842]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:57.138 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 6 on host 'smithi177' 2024-04-12T21:42:58.111 DEBUG:teuthology.orchestra.run.smithi177:osd.6> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.6.service 2024-04-12T21:42:58.113 INFO:tasks.cephadm:Deploying osd.7 on smithi177 with /dev/vg_nvme/lv_1... 2024-04-12T21:42:58.114 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-12T21:42:58.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:58 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:58.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:58 smithi123 ceph-mon[24842]: from='osd.6 [v2:172.21.15.177:6816/1659945166,v1:172.21.15.177:6817/1659945166]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T21:42:58.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:58 smithi123 ceph-mon[24842]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T21:42:58.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:58 smithi123 ceph-mon[24842]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:58.422 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:58 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:42:58.422 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:58 smithi177 ceph-mon[26903]: from='osd.6 [v2:172.21.15.177:6816/1659945166,v1:172.21.15.177:6817/1659945166]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T21:42:58.422 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:58 smithi177 ceph-mon[26903]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T21:42:58.422 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:58 smithi177 ceph-mon[26903]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:42:59.276 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:59 smithi177 ceph-mon[26903]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T21:42:59.276 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:59 smithi177 ceph-mon[26903]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T21:42:59.276 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:59 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:42:59.276 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:59 smithi177 ceph-mon[26903]: from='osd.6 [v2:172.21.15.177:6816/1659945166,v1:172.21.15.177:6817/1659945166]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:59.276 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:42:59 smithi177 ceph-mon[26903]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:59.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:42:59 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[36996]: 2024-04-12T21:42:59.149+0000 7f03fd8ff700 -1 osd.6 0 waiting for initial osdmap 2024-04-12T21:42:59.277 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:42:59 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[36996]: 2024-04-12T21:42:59.155+0000 7f03f7a95700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:42:59.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:59 smithi123 ceph-mon[24842]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T21:42:59.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:59 smithi123 ceph-mon[24842]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T21:42:59.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:59 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:42:59.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:59 smithi123 ceph-mon[24842]: from='osd.6 [v2:172.21.15.177:6816/1659945166,v1:172.21.15.177:6817/1659945166]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:42:59.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:42:59 smithi123 ceph-mon[24842]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:43:00.232 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:00 smithi177 ceph-mon[26903]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-04-12T21:43:00.232 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:00 smithi177 ceph-mon[26903]: osdmap e46: 7 total, 6 up, 7 in 2024-04-12T21:43:00.232 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:00 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:43:00.232 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:00 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:43:00.232 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:00 smithi177 ceph-mon[26903]: osd.6 [v2:172.21.15.177:6816/1659945166,v1:172.21.15.177:6817/1659945166] boot 2024-04-12T21:43:00.232 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:00 smithi177 ceph-mon[26903]: osdmap e47: 7 total, 7 up, 7 in 2024-04-12T21:43:00.232 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:00 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:43:00.233 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:00 smithi177 ceph-mon[26903]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:43:00.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:00 smithi123 ceph-mon[24842]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-04-12T21:43:00.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:00 smithi123 ceph-mon[24842]: osdmap e46: 7 total, 6 up, 7 in 2024-04-12T21:43:00.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:00 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:43:00.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:00 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:43:00.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:00 smithi123 ceph-mon[24842]: osd.6 [v2:172.21.15.177:6816/1659945166,v1:172.21.15.177:6817/1659945166] boot 2024-04-12T21:43:00.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:00 smithi123 ceph-mon[24842]: osdmap e47: 7 total, 7 up, 7 in 2024-04-12T21:43:00.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:00 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:43:00.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:00 smithi123 ceph-mon[24842]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T21:43:00.630 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-12T21:43:00.630 INFO:teuthology.orchestra.run.smithi177.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-12T21:43:00.630 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10+0 records in 2024-04-12T21:43:00.630 INFO:teuthology.orchestra.run.smithi177.stderr:10+0 records out 2024-04-12T21:43:00.631 INFO:teuthology.orchestra.run.smithi177.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135233 s, 775 MB/s 2024-04-12T21:43:00.631 INFO:teuthology.orchestra.run.smithi177.stderr:--> Zapping successful for: 2024-04-12T21:43:01.327 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi177:vg_nvme/lv_1 2024-04-12T21:43:01.557 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:01 smithi177 ceph-mon[26903]: purged_snaps scrub starts 2024-04-12T21:43:01.557 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:01 smithi177 ceph-mon[26903]: purged_snaps scrub ok 2024-04-12T21:43:01.557 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:01 smithi177 ceph-mon[26903]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T21:43:01.557 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:01 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:01 smithi123 ceph-mon[24842]: purged_snaps scrub starts 2024-04-12T21:43:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:01 smithi123 ceph-mon[24842]: purged_snaps scrub ok 2024-04-12T21:43:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:01 smithi123 ceph-mon[24842]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T21:43:01.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:01 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:02.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:02 smithi123 ceph-mon[24842]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T21:43:02.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:02 smithi123 ceph-mon[24842]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:02.715 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:02 smithi177 ceph-mon[26903]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T21:43:02.716 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:02 smithi177 ceph-mon[26903]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:03 smithi177 ceph-mon[26903]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T21:43:03.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:03 smithi123 ceph-mon[24842]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T21:43:04.485 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:04 smithi177 ceph-mon[26903]: osdmap e51: 7 total, 7 up, 7 in 2024-04-12T21:43:04.486 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:04 smithi177 ceph-mon[26903]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:04.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:04 smithi123 ceph-mon[24842]: osdmap e51: 7 total, 7 up, 7 in 2024-04-12T21:43:04.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:04 smithi123 ceph-mon[24842]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:05.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:43:05.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:43:05.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:05.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:05.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:05.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:05.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:05.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:05.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:05.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:43:05.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:05.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:43:05.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:05 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:05.695 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T21:43:05.695 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T21:43:05.695 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:05.695 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:05.695 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:05.696 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:05.696 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:05.696 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:05.696 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:05.696 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:43:05.696 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:05.696 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:43:05.696 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:05 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:06.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:06 smithi123 ceph-mon[24842]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:43:06.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:06 smithi123 ceph-mon[24842]: Detected new or changed devices on smithi177 2024-04-12T21:43:06.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:06 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi177 to 5312M 2024-04-12T21:43:06.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:06 smithi123 ceph-mon[24842]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-12T21:43:06.669 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:06 smithi177 ceph-mon[26903]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi177:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:43:06.669 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:06 smithi177 ceph-mon[26903]: Detected new or changed devices on smithi177 2024-04-12T21:43:06.669 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:06 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi177 to 5312M 2024-04-12T21:43:06.669 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:06 smithi177 ceph-mon[26903]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-12T21:43:07.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:07 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/4086793186' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e336f99d-92b4-40ab-9f64-69a745adab0c"}]: dispatch 2024-04-12T21:43:07.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:07 smithi123 ceph-mon[24842]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e336f99d-92b4-40ab-9f64-69a745adab0c"}]: dispatch 2024-04-12T21:43:07.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:07 smithi123 ceph-mon[24842]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e336f99d-92b4-40ab-9f64-69a745adab0c"}]': finished 2024-04-12T21:43:07.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:07 smithi123 ceph-mon[24842]: osdmap e52: 8 total, 7 up, 8 in 2024-04-12T21:43:07.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:07 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:07.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:07 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/4086793186' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e336f99d-92b4-40ab-9f64-69a745adab0c"}]: dispatch 2024-04-12T21:43:07.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:07 smithi177 ceph-mon[26903]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e336f99d-92b4-40ab-9f64-69a745adab0c"}]: dispatch 2024-04-12T21:43:07.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:07 smithi177 ceph-mon[26903]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e336f99d-92b4-40ab-9f64-69a745adab0c"}]': finished 2024-04-12T21:43:07.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:07 smithi177 ceph-mon[26903]: osdmap e52: 8 total, 7 up, 8 in 2024-04-12T21:43:07.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:07 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:08.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:08 smithi177 ceph-mon[26903]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T21:43:08.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:08 smithi177 ceph-mon[26903]: from='client.? 172.21.15.177:0/959617586' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:43:08.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:08 smithi123 ceph-mon[24842]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T21:43:08.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:08 smithi123 ceph-mon[24842]: from='client.? 172.21.15.177:0/959617586' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T21:43:10.481 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:10 smithi177 ceph-mon[26903]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-12T21:43:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:10 smithi123 ceph-mon[24842]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-12T21:43:12.767 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:12 smithi177 ceph-mon[26903]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-12T21:43:12.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:12 smithi123 ceph-mon[24842]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-12T21:43:14.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:14 smithi177 ceph-mon[26903]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T21:43:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:14 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T21:43:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:14 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:14 smithi177 ceph-mon[26903]: Deploying daemon osd.7 on smithi177 2024-04-12T21:43:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:14 smithi123 ceph-mon[24842]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T21:43:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:14 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T21:43:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:14 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:14.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:14 smithi123 ceph-mon[24842]: Deploying daemon osd.7 on smithi177 2024-04-12T21:43:16.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:16 smithi123 ceph-mon[24842]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:16.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:16 smithi177 ceph-mon[26903]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:18.650 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:18 smithi177 ceph-mon[26903]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:18.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:18 smithi123 ceph-mon[24842]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:20.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:19 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:20.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:19 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:43:20.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:19 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:20.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:19 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:43:20.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:19 smithi177 ceph-mon[26903]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:20.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:19 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:20.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:19 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:43:20.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:19 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:20.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:19 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:43:20.418 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:19 smithi123 ceph-mon[24842]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:21.791 INFO:teuthology.orchestra.run.smithi177.stdout:Created osd(s) 7 on host 'smithi177' 2024-04-12T21:43:22.435 DEBUG:teuthology.orchestra.run.smithi177:osd.7> sudo journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.7.service 2024-04-12T21:43:22.438 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-12T21:43:22.438 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd stat -f json 2024-04-12T21:43:22.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:22 smithi123 ceph-mon[24842]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:22.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:22.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:22 smithi123 ceph-mon[24842]: from='osd.7 [v2:172.21.15.177:6824/3189077372,v1:172.21.15.177:6825/3189077372]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T21:43:22.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:22 smithi123 ceph-mon[24842]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T21:43:22.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:22 smithi177 ceph-mon[26903]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:22.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:22.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:22 smithi177 ceph-mon[26903]: from='osd.7 [v2:172.21.15.177:6824/3189077372,v1:172.21.15.177:6825/3189077372]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T21:43:22.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:22 smithi177 ceph-mon[26903]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T21:43:24.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:23 smithi123 ceph-mon[24842]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T21:43:24.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:23 smithi123 ceph-mon[24842]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T21:43:24.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:24.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:23 smithi123 ceph-mon[24842]: from='osd.7 [v2:172.21.15.177:6824/3189077372,v1:172.21.15.177:6825/3189077372]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:43:24.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:23 smithi123 ceph-mon[24842]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:43:24.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:23 smithi123 ceph-mon[24842]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:24.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:43:24.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:43:24.244 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-mon[26903]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T21:43:24.244 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-mon[26903]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T21:43:24.244 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:24.244 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-mon[26903]: from='osd.7 [v2:172.21.15.177:6824/3189077372,v1:172.21.15.177:6825/3189077372]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:43:24.244 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-mon[26903]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:43:24.245 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-mon[26903]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T21:43:24.245 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:43:24.245 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:43:24.245 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[40083]: 2024-04-12T21:43:23.799+0000 7f23facde700 -1 osd.7 0 waiting for initial osdmap 2024-04-12T21:43:24.245 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:43:23 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[40083]: 2024-04-12T21:43:23.806+0000 7f23f5e76700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:43:24.484 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:43:25.073 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-04-12T21:43:25.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: osdmap e54: 8 total, 7 up, 8 in 2024-04-12T21:43:25.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:25.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:25.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:25.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: osd.7 [v2:172.21.15.177:6824/3189077372,v1:172.21.15.177:6825/3189077372] boot 2024-04-12T21:43:25.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: osdmap e55: 8 total, 8 up, 8 in 2024-04-12T21:43:25.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:25.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:24 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3679824954' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T21:43:25.098 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1712958204,"num_in_osds":8,"osd_in_since":1712958187,"num_remapped_pgs":1} 2024-04-12T21:43:25.099 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]': finished 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: osdmap e54: 8 total, 7 up, 8 in 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: osd.7 [v2:172.21.15.177:6824/3189077372,v1:172.21.15.177:6825/3189077372] boot 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: osdmap e55: 8 total, 8 up, 8 in 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:43:25.273 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:24 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3679824954' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T21:43:25.322 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:25.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:25 smithi123 ceph-mon[24842]: purged_snaps scrub starts 2024-04-12T21:43:25.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:25 smithi123 ceph-mon[24842]: purged_snaps scrub ok 2024-04-12T21:43:25.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:25 smithi123 ceph-mon[24842]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T21:43:25.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:25 smithi123 ceph-mon[24842]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:26.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:25 smithi177 ceph-mon[26903]: purged_snaps scrub starts 2024-04-12T21:43:26.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:25 smithi177 ceph-mon[26903]: purged_snaps scrub ok 2024-04-12T21:43:26.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:25 smithi177 ceph-mon[26903]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T21:43:26.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:25 smithi177 ceph-mon[26903]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:27.171 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:43:27.171 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":57,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","created":"2024-04-12T21:37:29.007719+0000","modified":"2024-04-12T21:43:26.391779+0000","last_up_change":"2024-04-12T21:43:24.385358+0000","last_in_change":"2024-04-12T21:43:07.369691+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T21:41:19.782617+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cb3f406e-c2cc-4356-8eb5-e9b82ac770b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6803","nonce":1737101118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6805","nonce":1737101118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6809","nonce":1737101118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6807","nonce":1737101118}]},"public_addr":"172.21.15.123:6803/1737101118","cluster_addr":"172.21.15.123:6805/1737101118","heartbeat_back_addr":"172.21.15.123:6809/1737101118","heartbeat_front_addr":"172.21.15.123:6807/1737101118","state":["exists","up"]},{"osd":1,"uuid":"f564e575-1d0c-4da6-bfdc-18c40fa666fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6811","nonce":1173353237}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6813","nonce":1173353237}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6817","nonce":1173353237}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6815","nonce":1173353237}]},"public_addr":"172.21.15.123:6811/1173353237","cluster_addr":"172.21.15.123:6813/1173353237","heartbeat_back_addr":"172.21.15.123:6817/1173353237","heartbeat_front_addr":"172.21.15.123:6815/1173353237","state":["exists","up"]},{"osd":2,"uuid":"66b74ab5-7a4e-4f96-a1ac-b5e4bf678d53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6819","nonce":1826006435}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6821","nonce":1826006435}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6825","nonce":1826006435}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6823","nonce":1826006435}]},"public_addr":"172.21.15.123:6819/1826006435","cluster_addr":"172.21.15.123:6821/1826006435","heartbeat_back_addr":"172.21.15.123:6825/1826006435","heartbeat_front_addr":"172.21.15.123:6823/1826006435","state":["exists","up"]},{"osd":3,"uuid":"25ce9222-04fa-4b49-9312-4567174dc425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6827","nonce":270637540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6829","nonce":270637540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6832","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6833","nonce":270637540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6831","nonce":270637540}]},"public_addr":"172.21.15.123:6827/270637540","cluster_addr":"172.21.15.123:6829/270637540","heartbeat_back_addr":"172.21.15.123:6833/270637540","heartbeat_front_addr":"172.21.15.123:6831/270637540","state":["exists","up"]},{"osd":4,"uuid":"e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6801","nonce":2901225367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6803","nonce":2901225367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6807","nonce":2901225367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6805","nonce":2901225367}]},"public_addr":"172.21.15.177:6801/2901225367","cluster_addr":"172.21.15.177:6803/2901225367","heartbeat_back_addr":"172.21.15.177:6807/2901225367","heartbeat_front_addr":"172.21.15.177:6805/2901225367","state":["exists","up"]},{"osd":5,"uuid":"77011f8f-279b-4bd1-a254-3d58ddaa0bbf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6809","nonce":3827733016}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6811","nonce":3827733016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6815","nonce":3827733016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6813","nonce":3827733016}]},"public_addr":"172.21.15.177:6809/3827733016","cluster_addr":"172.21.15.177:6811/3827733016","heartbeat_back_addr":"172.21.15.177:6815/3827733016","heartbeat_front_addr":"172.21.15.177:6813/3827733016","state":["exists","up"]},{"osd":6,"uuid":"5f0b81d8-73ce-4aef-95d1-c6216242457e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6817","nonce":1659945166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6819","nonce":1659945166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6823","nonce":1659945166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6821","nonce":1659945166}]},"public_addr":"172.21.15.177:6817/1659945166","cluster_addr":"172.21.15.177:6819/1659945166","heartbeat_back_addr":"172.21.15.177:6823/1659945166","heartbeat_front_addr":"172.21.15.177:6821/1659945166","state":["exists","up"]},{"osd":7,"uuid":"e336f99d-92b4-40ab-9f64-69a745adab0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6825","nonce":3189077372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6827","nonce":3189077372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6831","nonce":3189077372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6829","nonce":3189077372}]},"public_addr":"172.21.15.177:6825/3189077372","cluster_addr":"172.21.15.177:6827/3189077372","heartbeat_back_addr":"172.21.15.177:6831/3189077372","heartbeat_front_addr":"172.21.15.177:6829/3189077372","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:40:21.146135+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:40:50.264770+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:41:17.891000+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:41:47.272737+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:09.226392+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:33.841286+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:58.456071+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:43:23.115774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.123:0/3720783718":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6801/2676240919":"2024-04-13T21:39:23.419403+0000","172.21.15.123:0/1802393846":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/2186353003":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/268303400":"2024-04-13T21:38:31.689573+0000","172.21.15.123:6801/255734543":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/1830920697":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6800/2676240919":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6800/255734543":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/38853189":"2024-04-13T21:38:04.111148+0000","172.21.15.123:6800/4278471775":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/4001561915":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6801/4278471775":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/86186754":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/930511870":"2024-04-13T21:38:04.111148+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T21:43:27.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T21:43:27.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:27.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:27.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:27 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/176877473' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T21:43:27.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T21:43:27.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:43:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:27 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/176877473' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T21:43:27.871 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-12T21:41:19.782617+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-12T21:43:27.871 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-12T21:43:28.084 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:28.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:28 smithi123 ceph-mon[24842]: Detected new or changed devices on smithi177 2024-04-12T21:43:28.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:28 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi177 to 3984M 2024-04-12T21:43:28.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:28 smithi123 ceph-mon[24842]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T21:43:28.491 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:28 smithi123 ceph-mon[24842]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:28.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:28 smithi177 ceph-mon[26903]: Detected new or changed devices on smithi177 2024-04-12T21:43:28.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:28 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi177 to 3984M 2024-04-12T21:43:28.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:28 smithi177 ceph-mon[26903]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T21:43:28.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:28 smithi177 ceph-mon[26903]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:29.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:29 smithi123 ceph-mon[24842]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T21:43:29.708 INFO:teuthology.orchestra.run.smithi123.stdout:pg_num: 1 2024-04-12T21:43:29.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:29 smithi177 ceph-mon[26903]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T21:43:30.387 INFO:tasks.cephadm:Setting up client nodes... 2024-04-12T21:43:30.387 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-12T21:43:30.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:30 smithi123 ceph-mon[24842]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:30.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:30 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/579231370' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T21:43:30.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:30 smithi177 ceph-mon[26903]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:30.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:30 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/579231370' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T21:43:32.372 INFO:teuthology.orchestra.run.smithi123.stdout:[client.0] 2024-04-12T21:43:32.372 INFO:teuthology.orchestra.run.smithi123.stdout: key = AQAEqxlmGo/6FRAAeO25LRbCMz2JzJYiUGsYhQ== 2024-04-12T21:43:32.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:32 smithi123 ceph-mon[24842]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T21:43:32.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:32 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1084980308' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T21:43:32.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:32 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1084980308' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T21:43:33.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:32 smithi177 ceph-mon[26903]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T21:43:33.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:32 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1084980308' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T21:43:33.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:32 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1084980308' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T21:43:33.060 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-04-12T21:43:33.060 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-12T21:43:33.060 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-12T21:43:33.103 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-12T21:43:33.103 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-12T21:43:33.103 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph mgr dump --format=json 2024-04-12T21:43:33.358 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:34.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:34 smithi123 ceph-mon[24842]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-12T21:43:35.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:34 smithi177 ceph-mon[26903]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-12T21:43:35.249 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:43:35.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:35 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/104576208' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T21:43:35.961 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi123.psjrfs","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":2291483362},{"type":"v1","addr":"172.21.15.123:6801","nonce":2291483362}]},"active_addr":"172.21.15.123:6801/2291483362","active_change":"2024-04-12T21:39:23.419695+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi177.whybza","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.123:8443/","prometheus":"http://172.21.15.123:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.123:0","nonce":1967576326}]},{"addrvec":[{"type":"v2","addr":"172.21.15.123:0","nonce":511713245}]},{"addrvec":[{"type":"v2","addr":"172.21.15.123:0","nonce":751524146}]},{"addrvec":[{"type":"v2","addr":"172.21.15.123:0","nonce":1167029155}]}]}} 2024-04-12T21:43:35.966 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-12T21:43:35.967 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T21:43:35.967 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-12T21:43:36.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:35 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/104576208' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T21:43:36.179 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:36.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:36 smithi123 ceph-mon[24842]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-12T21:43:37.024 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:36 smithi177 ceph-mon[26903]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-04-12T21:43:38.100 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:43:38.100 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":59,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","created":"2024-04-12T21:37:29.007719+0000","modified":"2024-04-12T21:43:28.407949+0000","last_up_change":"2024-04-12T21:43:24.385358+0000","last_in_change":"2024-04-12T21:43:07.369691+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T21:41:19.782617+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cb3f406e-c2cc-4356-8eb5-e9b82ac770b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6803","nonce":1737101118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6805","nonce":1737101118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6809","nonce":1737101118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6807","nonce":1737101118}]},"public_addr":"172.21.15.123:6803/1737101118","cluster_addr":"172.21.15.123:6805/1737101118","heartbeat_back_addr":"172.21.15.123:6809/1737101118","heartbeat_front_addr":"172.21.15.123:6807/1737101118","state":["exists","up"]},{"osd":1,"uuid":"f564e575-1d0c-4da6-bfdc-18c40fa666fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6811","nonce":1173353237}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6813","nonce":1173353237}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6817","nonce":1173353237}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6815","nonce":1173353237}]},"public_addr":"172.21.15.123:6811/1173353237","cluster_addr":"172.21.15.123:6813/1173353237","heartbeat_back_addr":"172.21.15.123:6817/1173353237","heartbeat_front_addr":"172.21.15.123:6815/1173353237","state":["exists","up"]},{"osd":2,"uuid":"66b74ab5-7a4e-4f96-a1ac-b5e4bf678d53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6819","nonce":1826006435}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6821","nonce":1826006435}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6825","nonce":1826006435}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6823","nonce":1826006435}]},"public_addr":"172.21.15.123:6819/1826006435","cluster_addr":"172.21.15.123:6821/1826006435","heartbeat_back_addr":"172.21.15.123:6825/1826006435","heartbeat_front_addr":"172.21.15.123:6823/1826006435","state":["exists","up"]},{"osd":3,"uuid":"25ce9222-04fa-4b49-9312-4567174dc425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6827","nonce":270637540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6829","nonce":270637540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6832","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6833","nonce":270637540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6831","nonce":270637540}]},"public_addr":"172.21.15.123:6827/270637540","cluster_addr":"172.21.15.123:6829/270637540","heartbeat_back_addr":"172.21.15.123:6833/270637540","heartbeat_front_addr":"172.21.15.123:6831/270637540","state":["exists","up"]},{"osd":4,"uuid":"e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6801","nonce":2901225367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6803","nonce":2901225367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6807","nonce":2901225367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6805","nonce":2901225367}]},"public_addr":"172.21.15.177:6801/2901225367","cluster_addr":"172.21.15.177:6803/2901225367","heartbeat_back_addr":"172.21.15.177:6807/2901225367","heartbeat_front_addr":"172.21.15.177:6805/2901225367","state":["exists","up"]},{"osd":5,"uuid":"77011f8f-279b-4bd1-a254-3d58ddaa0bbf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6809","nonce":3827733016}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6811","nonce":3827733016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6815","nonce":3827733016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6813","nonce":3827733016}]},"public_addr":"172.21.15.177:6809/3827733016","cluster_addr":"172.21.15.177:6811/3827733016","heartbeat_back_addr":"172.21.15.177:6815/3827733016","heartbeat_front_addr":"172.21.15.177:6813/3827733016","state":["exists","up"]},{"osd":6,"uuid":"5f0b81d8-73ce-4aef-95d1-c6216242457e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6817","nonce":1659945166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6819","nonce":1659945166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6823","nonce":1659945166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6821","nonce":1659945166}]},"public_addr":"172.21.15.177:6817/1659945166","cluster_addr":"172.21.15.177:6819/1659945166","heartbeat_back_addr":"172.21.15.177:6823/1659945166","heartbeat_front_addr":"172.21.15.177:6821/1659945166","state":["exists","up"]},{"osd":7,"uuid":"e336f99d-92b4-40ab-9f64-69a745adab0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6825","nonce":3189077372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6827","nonce":3189077372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6831","nonce":3189077372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6829","nonce":3189077372}]},"public_addr":"172.21.15.177:6825/3189077372","cluster_addr":"172.21.15.177:6827/3189077372","heartbeat_back_addr":"172.21.15.177:6831/3189077372","heartbeat_front_addr":"172.21.15.177:6829/3189077372","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:40:21.146135+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:40:50.264770+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:41:17.891000+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:41:47.272737+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:09.226392+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:33.841286+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:58.456071+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:43:23.115774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.123:0/3720783718":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6801/2676240919":"2024-04-13T21:39:23.419403+0000","172.21.15.123:0/1802393846":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/2186353003":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/268303400":"2024-04-13T21:38:31.689573+0000","172.21.15.123:6801/255734543":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/1830920697":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6800/2676240919":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6800/255734543":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/38853189":"2024-04-13T21:38:04.111148+0000","172.21.15.123:6800/4278471775":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/4001561915":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6801/4278471775":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/86186754":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/930511870":"2024-04-13T21:38:04.111148+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T21:43:38.777 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T21:43:38.778 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-12T21:43:38.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:38 smithi123 ceph-mon[24842]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T21:43:38.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:38 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3855418565' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T21:43:38.989 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:39.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:38 smithi177 ceph-mon[26903]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T21:43:39.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:38 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3855418565' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T21:43:40.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:40 smithi123 ceph-mon[24842]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-04-12T21:43:40.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:40 smithi177 ceph-mon[26903]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-04-12T21:43:40.739 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:43:40.740 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":59,"fsid":"c8b0510a-f914-11ee-bc8d-c7b262605968","created":"2024-04-12T21:37:29.007719+0000","modified":"2024-04-12T21:43:28.407949+0000","last_up_change":"2024-04-12T21:43:24.385358+0000","last_in_change":"2024-04-12T21:43:07.369691+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T21:41:19.782617+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"cb3f406e-c2cc-4356-8eb5-e9b82ac770b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6803","nonce":1737101118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6805","nonce":1737101118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6809","nonce":1737101118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":1737101118},{"type":"v1","addr":"172.21.15.123:6807","nonce":1737101118}]},"public_addr":"172.21.15.123:6803/1737101118","cluster_addr":"172.21.15.123:6805/1737101118","heartbeat_back_addr":"172.21.15.123:6809/1737101118","heartbeat_front_addr":"172.21.15.123:6807/1737101118","state":["exists","up"]},{"osd":1,"uuid":"f564e575-1d0c-4da6-bfdc-18c40fa666fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6811","nonce":1173353237}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6813","nonce":1173353237}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6817","nonce":1173353237}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":1173353237},{"type":"v1","addr":"172.21.15.123:6815","nonce":1173353237}]},"public_addr":"172.21.15.123:6811/1173353237","cluster_addr":"172.21.15.123:6813/1173353237","heartbeat_back_addr":"172.21.15.123:6817/1173353237","heartbeat_front_addr":"172.21.15.123:6815/1173353237","state":["exists","up"]},{"osd":2,"uuid":"66b74ab5-7a4e-4f96-a1ac-b5e4bf678d53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6819","nonce":1826006435}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6821","nonce":1826006435}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6825","nonce":1826006435}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":1826006435},{"type":"v1","addr":"172.21.15.123:6823","nonce":1826006435}]},"public_addr":"172.21.15.123:6819/1826006435","cluster_addr":"172.21.15.123:6821/1826006435","heartbeat_back_addr":"172.21.15.123:6825/1826006435","heartbeat_front_addr":"172.21.15.123:6823/1826006435","state":["exists","up"]},{"osd":3,"uuid":"25ce9222-04fa-4b49-9312-4567174dc425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6827","nonce":270637540}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6829","nonce":270637540}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6832","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6833","nonce":270637540}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":270637540},{"type":"v1","addr":"172.21.15.123:6831","nonce":270637540}]},"public_addr":"172.21.15.123:6827/270637540","cluster_addr":"172.21.15.123:6829/270637540","heartbeat_back_addr":"172.21.15.123:6833/270637540","heartbeat_front_addr":"172.21.15.123:6831/270637540","state":["exists","up"]},{"osd":4,"uuid":"e2b8d83d-77b1-4c8a-bed8-788a1c0bb50a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6800","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6801","nonce":2901225367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6802","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6803","nonce":2901225367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6806","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6807","nonce":2901225367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6804","nonce":2901225367},{"type":"v1","addr":"172.21.15.177:6805","nonce":2901225367}]},"public_addr":"172.21.15.177:6801/2901225367","cluster_addr":"172.21.15.177:6803/2901225367","heartbeat_back_addr":"172.21.15.177:6807/2901225367","heartbeat_front_addr":"172.21.15.177:6805/2901225367","state":["exists","up"]},{"osd":5,"uuid":"77011f8f-279b-4bd1-a254-3d58ddaa0bbf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6808","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6809","nonce":3827733016}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6810","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6811","nonce":3827733016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6814","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6815","nonce":3827733016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6812","nonce":3827733016},{"type":"v1","addr":"172.21.15.177:6813","nonce":3827733016}]},"public_addr":"172.21.15.177:6809/3827733016","cluster_addr":"172.21.15.177:6811/3827733016","heartbeat_back_addr":"172.21.15.177:6815/3827733016","heartbeat_front_addr":"172.21.15.177:6813/3827733016","state":["exists","up"]},{"osd":6,"uuid":"5f0b81d8-73ce-4aef-95d1-c6216242457e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6816","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6817","nonce":1659945166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6818","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6819","nonce":1659945166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6822","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6823","nonce":1659945166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6820","nonce":1659945166},{"type":"v1","addr":"172.21.15.177:6821","nonce":1659945166}]},"public_addr":"172.21.15.177:6817/1659945166","cluster_addr":"172.21.15.177:6819/1659945166","heartbeat_back_addr":"172.21.15.177:6823/1659945166","heartbeat_front_addr":"172.21.15.177:6821/1659945166","state":["exists","up"]},{"osd":7,"uuid":"e336f99d-92b4-40ab-9f64-69a745adab0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6824","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6825","nonce":3189077372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6826","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6827","nonce":3189077372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6830","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6831","nonce":3189077372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.177:6828","nonce":3189077372},{"type":"v1","addr":"172.21.15.177:6829","nonce":3189077372}]},"public_addr":"172.21.15.177:6825/3189077372","cluster_addr":"172.21.15.177:6827/3189077372","heartbeat_back_addr":"172.21.15.177:6831/3189077372","heartbeat_front_addr":"172.21.15.177:6829/3189077372","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:40:21.146135+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:40:50.264770+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:41:17.891000+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:41:47.272737+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:09.226392+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:33.841286+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:42:58.456071+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T21:43:23.115774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.123:0/3720783718":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6801/2676240919":"2024-04-13T21:39:23.419403+0000","172.21.15.123:0/1802393846":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/2186353003":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/268303400":"2024-04-13T21:38:31.689573+0000","172.21.15.123:6801/255734543":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/1830920697":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6800/2676240919":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6800/255734543":"2024-04-13T21:38:31.689573+0000","172.21.15.123:0/38853189":"2024-04-13T21:38:04.111148+0000","172.21.15.123:6800/4278471775":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/4001561915":"2024-04-13T21:39:23.419403+0000","172.21.15.123:6801/4278471775":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/86186754":"2024-04-13T21:38:04.111148+0000","172.21.15.123:0/930511870":"2024-04-13T21:38:04.111148+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T21:43:41.400 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T21:43:41.400 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T21:43:41.401 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T21:43:41.401 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T21:43:41.402 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T21:43:41.402 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T21:43:41.403 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T21:43:41.403 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T21:43:41.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:41 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2331970227' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T21:43:41.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:41 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2331970227' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T21:43:41.818 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:41.821 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:41.821 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:41.822 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:41.826 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:41.828 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:41.839 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:41.841 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:42.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:42 smithi177 ceph-mon[26903]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-12T21:43:42.557 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:42 smithi123 ceph-mon[24842]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-12T21:43:44.897 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:44 smithi123 ceph-mon[24842]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:45.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:44 smithi177 ceph-mon[26903]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:46.658 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:46 smithi123 ceph-mon[24842]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:47.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:46 smithi177 ceph-mon[26903]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:48 smithi123 ceph-mon[24842]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:49.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:48 smithi177 ceph-mon[26903]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:50.657 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:50 smithi123 ceph-mon[24842]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:50.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:50 smithi177 ceph-mon[26903]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:52.685 INFO:teuthology.orchestra.run.smithi123.stdout:81604378656 2024-04-12T21:43:52.685 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T21:43:52.710 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:52 smithi123 ceph-mon[24842]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:53.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:52 smithi177 ceph-mon[26903]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:54.035 INFO:teuthology.orchestra.run.smithi123.stdout:38654705708 2024-04-12T21:43:54.036 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T21:43:54.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:54 smithi123 ceph-mon[24842]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:55.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:54 smithi177 ceph-mon[26903]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:55.873 INFO:teuthology.orchestra.run.smithi123.stdout:60129542184 2024-04-12T21:43:55.873 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T21:43:56.505 INFO:teuthology.orchestra.run.smithi123.stdout:236223201287 2024-04-12T21:43:56.506 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T21:43:56.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:56 smithi123 ceph-mon[24842]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:56.886 INFO:teuthology.orchestra.run.smithi123.stdout:133143986200 2024-04-12T21:43:56.886 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T21:43:57.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:56 smithi177 ceph-mon[26903]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:57.937 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:57.958 INFO:teuthology.orchestra.run.smithi123.stdout:167503724563 2024-04-12T21:43:57.959 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T21:43:58.831 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:43:58 smithi123 ceph-mon[24842]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:43:58 smithi177 ceph-mon[26903]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:43:59.397 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:59.398 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:59.398 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:59.399 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:43:59.911 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:44:00.470 INFO:teuthology.orchestra.run.smithi123.stdout:201863462926 2024-04-12T21:44:00.471 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T21:44:00.660 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:00 smithi123 ceph-mon[24842]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:00.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:00 smithi177 ceph-mon[26903]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:00.965 INFO:teuthology.orchestra.run.smithi123.stdout:111669149725 2024-04-12T21:44:00.966 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T21:44:02.630 INFO:teuthology.orchestra.run.smithi123.stdout:81604378659 2024-04-12T21:44:02.871 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:02 smithi123 ceph-mon[24842]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:03.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:02 smithi177 ceph-mon[26903]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:03.887 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:03 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3259177265' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T21:44:04.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:03 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3259177265' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T21:44:04.752 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378656 got 81604378659 for osd.2 2024-04-12T21:44:04.752 DEBUG:teuthology.parallel:result is None 2024-04-12T21:44:04.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:04 smithi123 ceph-mon[24842]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:04 smithi177 ceph-mon[26903]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:05.061 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:44:05.688 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:44:06.274 INFO:teuthology.orchestra.run.smithi123.stdout:60129542186 2024-04-12T21:44:06.907 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:06 smithi123 ceph-mon[24842]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:06.908 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:06 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/422590129' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T21:44:06.931 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542186 for osd.1 2024-04-12T21:44:06.931 DEBUG:teuthology.parallel:result is None 2024-04-12T21:44:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:06 smithi177 ceph-mon[26903]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:06 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/422590129' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T21:44:07.942 INFO:teuthology.orchestra.run.smithi123.stdout:133143986203 2024-04-12T21:44:08.130 INFO:teuthology.orchestra.run.smithi123.stdout:236223201291 2024-04-12T21:44:08.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:08 smithi123 ceph-mon[24842]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:08.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:08 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3177425345' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T21:44:08.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:08 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2130063280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T21:44:08.713 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986200 got 133143986203 for osd.4 2024-04-12T21:44:08.713 DEBUG:teuthology.parallel:result is None 2024-04-12T21:44:09.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:08 smithi177 ceph-mon[26903]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:08 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3177425345' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T21:44:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:08 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2130063280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T21:44:10.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:10 smithi123 ceph-mon[24842]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:10.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:10 smithi177 ceph-mon[26903]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:11.772 INFO:teuthology.orchestra.run.smithi123.stdout:167503724566 2024-04-12T21:44:12.069 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201287 got 236223201291 for osd.7 2024-04-12T21:44:12.069 DEBUG:teuthology.parallel:result is None 2024-04-12T21:44:12.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:12 smithi123 ceph-mon[24842]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:12.615 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:12 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2956665413' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T21:44:13.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:12 smithi177 ceph-mon[26903]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:13.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:12 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2956665413' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T21:44:13.347 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724563 got 167503724566 for osd.5 2024-04-12T21:44:13.347 DEBUG:teuthology.parallel:result is None 2024-04-12T21:44:14.523 INFO:teuthology.orchestra.run.smithi123.stdout:201863462929 2024-04-12T21:44:14.911 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:14 smithi123 ceph-mon[24842]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:14.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:14 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2310967207' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T21:44:15.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:14 smithi177 ceph-mon[26903]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:15.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:14 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2310967207' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T21:44:15.107 INFO:teuthology.orchestra.run.smithi123.stdout:38654705713 2024-04-12T21:44:15.800 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705713 for osd.0 2024-04-12T21:44:15.801 DEBUG:teuthology.parallel:result is None 2024-04-12T21:44:15.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:15 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/4164757062' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T21:44:16.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:15 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/4164757062' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T21:44:16.221 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462929 for osd.6 2024-04-12T21:44:16.222 DEBUG:teuthology.parallel:result is None 2024-04-12T21:44:16.826 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:16 smithi123 ceph-mon[24842]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:16.826 INFO:teuthology.orchestra.run.smithi123.stdout:111669149729 2024-04-12T21:44:17.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:16 smithi177 ceph-mon[26903]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:17.462 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149729 for osd.3 2024-04-12T21:44:17.462 DEBUG:teuthology.parallel:result is None 2024-04-12T21:44:17.462 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T21:44:17.463 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-12T21:44:17.684 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:44:17.805 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:17 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2394556789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T21:44:18.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:17 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2394556789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T21:44:18.815 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:18 smithi123 ceph-mon[24842]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:19.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:18 smithi177 ceph-mon[26903]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:19.429 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:44:19.430 INFO:teuthology.orchestra.run.smithi123.stderr:dumped all 2024-04-12T21:44:20.128 INFO:teuthology.orchestra.run.smithi123.stdout:{"pg_ready":true,"pg_map":{"version":194,"stamp":"2024-04-12T21:44:17.527749+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48812,"kb_used_data":6316,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912404,"statfs":{"total":767960285184,"available":767910301696,"internally_reserved":0,"allocated":6467584,"data_stored":3547737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002237"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-12T21:43:28.423509+0000","last_change":"2024-04-12T21:43:28.423509+0000","last_active":"2024-04-12T21:43:28.423509+0000","last_peered":"2024-04-12T21:43:28.423509+0000","last_clean":"2024-04-12T21:43:28.423509+0000","last_became_active":"2024-04-12T21:43:28.423025+0000","last_became_peered":"2024-04-12T21:43:28.423025+0000","last_unstale":"2024-04-12T21:43:28.423509+0000","last_undegraded":"2024-04-12T21:43:28.423509+0000","last_fullsized":"2024-04-12T21:43:28.423509+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T21:41:20.526399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T21:41:20.526399+0000","last_clean_scrub_stamp":"2024-04-12T21:41:20.526399+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:54:40.001218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40799999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53600000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59399999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56100000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.55100000000000005}]},{"osd":1,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.60599999999999998}]},{"osd":2,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.218,"5min":7.218,"15min":7.218},"last":0.47899999999999998}]},{"osd":3,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.2329999999999997,"5min":7.2329999999999997,"15min":7.2329999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.52500000000000002}]},{"osd":4,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.375},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.38700000000000001}]},{"osd":5,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.499},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.54200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:44:06 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.316,"15min":0.29999999999999999},"min":{"1min":0.252,"5min":0.247,"15min":0.247},"max":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.32900000000000001,"15min":0.311},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"last":0.35699999999999998}]},{"osd":2,"last update":"Fri Apr 12 21:43:28 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.35199999999999998,"15min":0.34799999999999998},"min":{"1min":0.22500000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"last":0.373},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.36599999999999999,"15min":0.36299999999999999},"min":{"1min":0.246,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.49299999999999999,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.38200000000000001}]},{"osd":3,"last update":"Fri Apr 12 21:43:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.439,"5min":0.44700000000000001,"15min":0.44800000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.48499999999999999}]},{"osd":4,"last update":"Fri Apr 12 21:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.76700000000000002}]},{"osd":5,"last update":"Fri Apr 12 21:43:36 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.628}]},{"osd":6,"last update":"Fri Apr 12 21:44:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.57199999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 21:44:00 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.316,"15min":0.28799999999999998},"min":{"1min":0.27000000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.32300000000000001,"15min":0.29199999999999998},"min":{"1min":0.26700000000000002,"5min":0.186,"15min":0.186},"max":{"1min":0.52600000000000002,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.317}]},{"osd":2,"last update":"Fri Apr 12 21:43:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.35899999999999999,"15min":0.34899999999999998},"min":{"1min":0.33600000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.38300000000000001,"15min":0.375},"min":{"1min":0.315,"5min":0.245,"15min":0.245},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.34000000000000002}]},{"osd":3,"last update":"Fri Apr 12 21:43:55 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.41299999999999998,"15min":0.41099999999999998},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.437,"15min":0.438},"min":{"1min":0.309,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.377}]},{"osd":4,"last update":"Fri Apr 12 21:43:12 2024","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.355,"15min":1.355},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":15.323,"5min":15.323,"15min":15.323},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.45200000000000001}]},{"osd":5,"last update":"Fri Apr 12 21:43:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.51100000000000001}]},{"osd":6,"last update":"Fri Apr 12 21:44:00 2024","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.433},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.499}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:43:22 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.34599999999999997,"15min":0.34000000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.35299999999999998,"15min":0.34499999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53900000000000003,"5min":0.54600000000000004,"15min":0.54600000000000004},"last":0.34200000000000003}]},{"osd":1,"last update":"Fri Apr 12 21:43:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.34399999999999997,"15min":0.33200000000000002},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.373,"15min":0.36099999999999999},"min":{"1min":0.30099999999999999,"5min":0.252,"15min":0.252},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.32800000000000001}]},{"osd":3,"last update":"Fri Apr 12 21:43:53 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.41599999999999998,"15min":0.40999999999999998},"min":{"1min":0.34399999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.441,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.379}]},{"osd":4,"last update":"Fri Apr 12 21:43:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.81499999999999995}]},{"osd":5,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.75600000000000001}]},{"osd":6,"last update":"Fri Apr 12 21:44:04 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.879}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76700000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:43:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.378,"15min":0.375},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.54400000000000004,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.29799999999999999}]},{"osd":1,"last update":"Fri Apr 12 21:43:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.27400000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.33700000000000002}]},{"osd":2,"last update":"Fri Apr 12 21:43:51 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.436},{"interface":"front","average":{"1min":0.44,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.32900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.34699999999999998}]},{"osd":4,"last update":"Fri Apr 12 21:43:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.51700000000000002}]},{"osd":5,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.57099999999999995}]},{"osd":6,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.60799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.54600000000000004,"15min":0.53700000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.442},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.56100000000000005,"15min":0.55600000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.46500000000000002}]},{"osd":1,"last update":"Fri Apr 12 21:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.495},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.627}]},{"osd":2,"last update":"Fri Apr 12 21:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.53900000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.52900000000000003,"15min":0.52100000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57699999999999996}]},{"osd":3,"last update":"Fri Apr 12 21:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.52800000000000002,"15min":0.51800000000000002},"min":{"1min":0.42899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54100000000000004,"15min":0.53500000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.71699999999999997}]},{"osd":5,"last update":"Fri Apr 12 21:43:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.77000000000000002}]},{"osd":6,"last update":"Fri Apr 12 21:43:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.75}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80800000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.52900000000000003}]},{"osd":1,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.61699999999999999}]},{"osd":2,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.53700000000000003}]},{"osd":3,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.47999999999999998}]},{"osd":4,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.37}]},{"osd":6,"last update":"Fri Apr 12 21:44:01 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.499}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T21:44:20.129 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-12T21:44:20.354 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:44:20.658 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:20 smithi123 ceph-mon[24842]: from='client.14446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:20.658 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:20 smithi123 ceph-mon[24842]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:20.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:20 smithi177 ceph-mon[26903]: from='client.14446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:20.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:20 smithi177 ceph-mon[26903]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:22.070 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:44:22.072 INFO:teuthology.orchestra.run.smithi123.stderr:dumped all 2024-04-12T21:44:22.667 INFO:teuthology.orchestra.run.smithi123.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2024-04-12T21:44:21.528367+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48812,"kb_used_data":6316,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912404,"statfs":{"total":767960285184,"available":767910301696,"internally_reserved":0,"allocated":6467584,"data_stored":3547737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001769"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-12T21:43:28.423509+0000","last_change":"2024-04-12T21:43:28.423509+0000","last_active":"2024-04-12T21:43:28.423509+0000","last_peered":"2024-04-12T21:43:28.423509+0000","last_clean":"2024-04-12T21:43:28.423509+0000","last_became_active":"2024-04-12T21:43:28.423025+0000","last_became_peered":"2024-04-12T21:43:28.423025+0000","last_unstale":"2024-04-12T21:43:28.423509+0000","last_undegraded":"2024-04-12T21:43:28.423509+0000","last_fullsized":"2024-04-12T21:43:28.423509+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T21:41:20.526399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T21:41:20.526399+0000","last_clean_scrub_stamp":"2024-04-12T21:41:20.526399+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-13T23:54:40.001218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69999999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68799999999999994}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63300000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60899999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.438}]},{"osd":1,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.40100000000000002}]},{"osd":2,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.218,"5min":7.218,"15min":7.218},"last":0.5}]},{"osd":3,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.2329999999999997,"5min":7.2329999999999997,"15min":7.2329999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.53200000000000003}]},{"osd":4,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.57999999999999996}]},{"osd":5,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.55800000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:44:06 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.316,"15min":0.29999999999999999},"min":{"1min":0.252,"5min":0.247,"15min":0.247},"max":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.32900000000000001,"15min":0.311},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"last":0.58099999999999996}]},{"osd":2,"last update":"Fri Apr 12 21:43:28 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.35199999999999998,"15min":0.34799999999999998},"min":{"1min":0.22500000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.36599999999999999,"15min":0.36299999999999999},"min":{"1min":0.246,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.49299999999999999,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.52400000000000002}]},{"osd":3,"last update":"Fri Apr 12 21:43:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.439,"5min":0.44700000000000001,"15min":0.44800000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.55400000000000005}]},{"osd":4,"last update":"Fri Apr 12 21:44:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55800000000000005,"15min":0.55200000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.56100000000000005,"15min":0.55500000000000005},"min":{"1min":0.442,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.82099999999999995}]},{"osd":5,"last update":"Fri Apr 12 21:43:36 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.73499999999999999}]},{"osd":6,"last update":"Fri Apr 12 21:44:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.97199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76300000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 21:44:00 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.316,"15min":0.28799999999999998},"min":{"1min":0.27000000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.32300000000000001,"15min":0.29199999999999998},"min":{"1min":0.26700000000000002,"5min":0.186,"15min":0.186},"max":{"1min":0.52600000000000002,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.29099999999999998}]},{"osd":2,"last update":"Fri Apr 12 21:43:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.35899999999999999,"15min":0.34899999999999998},"min":{"1min":0.33600000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.436},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.38300000000000001,"15min":0.375},"min":{"1min":0.315,"5min":0.245,"15min":0.245},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.32300000000000001}]},{"osd":3,"last update":"Fri Apr 12 21:43:55 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.41299999999999998,"15min":0.41099999999999998},"min":{"1min":0.30399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.437,"15min":0.438},"min":{"1min":0.309,"5min":0.308,"15min":0.308},"max":{"1min":0.60999999999999999,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.34999999999999998}]},{"osd":4,"last update":"Fri Apr 12 21:44:14 2024","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.4279999999999999,"15min":1.379},"min":{"1min":0.46700000000000003,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":17.908000000000001,"5min":17.908000000000001,"15min":17.908000000000001},"last":17.908000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.58499999999999996,"15min":0.57599999999999996},"min":{"1min":0.41999999999999998,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.64000000000000001}]},{"osd":5,"last update":"Fri Apr 12 21:43:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.59199999999999997}]},{"osd":6,"last update":"Fri Apr 12 21:44:00 2024","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.55300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:43:22 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.34599999999999997,"15min":0.34000000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.35299999999999998,"15min":0.34499999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53900000000000003,"5min":0.54600000000000004,"15min":0.54600000000000004},"last":0.34499999999999997}]},{"osd":1,"last update":"Fri Apr 12 21:43:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.34399999999999997,"15min":0.33200000000000002},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.373,"15min":0.36099999999999999},"min":{"1min":0.30099999999999999,"5min":0.252,"15min":0.252},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.33300000000000002}]},{"osd":3,"last update":"Fri Apr 12 21:43:53 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.41599999999999998,"15min":0.40999999999999998},"min":{"1min":0.34399999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.313},{"interface":"front","average":{"1min":0.441,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.32000000000000001}]},{"osd":4,"last update":"Fri Apr 12 21:44:16 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.58899999999999997,"15min":0.57899999999999996},"min":{"1min":0.46000000000000002,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.59899999999999998,"15min":0.59099999999999997},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.60799999999999998}]},{"osd":5,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.54900000000000004}]},{"osd":6,"last update":"Fri Apr 12 21:44:04 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.64000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55800000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:43:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.378,"15min":0.375},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.54400000000000004,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.49099999999999999}]},{"osd":1,"last update":"Fri Apr 12 21:43:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.314},{"interface":"front","average":{"1min":0.437,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.27400000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.35999999999999999}]},{"osd":2,"last update":"Fri Apr 12 21:43:51 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.32900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.41899999999999998}]},{"osd":4,"last update":"Fri Apr 12 21:44:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.56599999999999995}]},{"osd":5,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.40699999999999997}]},{"osd":6,"last update":"Fri Apr 12 21:44:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.47899999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.54600000000000004,"15min":0.53700000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.442},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.56100000000000005,"15min":0.55600000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.46500000000000002}]},{"osd":1,"last update":"Fri Apr 12 21:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.495},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.627}]},{"osd":2,"last update":"Fri Apr 12 21:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.53900000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.52900000000000003,"15min":0.52100000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57699999999999996}]},{"osd":3,"last update":"Fri Apr 12 21:44:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.52800000000000002,"15min":0.51800000000000002},"min":{"1min":0.42899999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54100000000000004,"15min":0.53500000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.71699999999999997}]},{"osd":5,"last update":"Fri Apr 12 21:43:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.77000000000000002}]},{"osd":6,"last update":"Fri Apr 12 21:43:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.75}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80800000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.52900000000000003}]},{"osd":1,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.61699999999999999}]},{"osd":2,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.53700000000000003}]},{"osd":3,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.47999999999999998}]},{"osd":4,"last update":"Fri Apr 12 21:43:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.37}]},{"osd":6,"last update":"Fri Apr 12 21:44:01 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.499}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T21:44:22.668 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T21:44:22.668 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-12T21:44:22.669 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-12T21:44:22.669 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-12T21:44:22.883 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/mon.smithi123/config 2024-04-12T21:44:22.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:22 smithi123 ceph-mon[24842]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:23.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:22 smithi177 ceph-mon[26903]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:23.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:23 smithi123 ceph-mon[24842]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:23.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:44:23.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:44:24.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:23 smithi177 ceph-mon[26903]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:44:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:44:24.595 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:44:24.596 INFO:teuthology.orchestra.run.smithi123.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-12T21:44:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:24 smithi123 ceph-mon[24842]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:24 smithi177 ceph-mon[26903]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:25.247 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-12T21:44:25.247 INFO:tasks.cephadm:Setup complete, yielding 2024-04-12T21:44:25.247 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T21:44:25.257 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:44:25.258 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch status' 2024-04-12T21:44:25.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:25 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3271764760' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T21:44:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:25 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3271764760' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T21:44:26.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:26 smithi123 ceph-mon[24842]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:27.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:26 smithi177 ceph-mon[26903]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:27.242 INFO:teuthology.orchestra.run.smithi123.stdout:Backend: cephadm 2024-04-12T21:44:27.242 INFO:teuthology.orchestra.run.smithi123.stdout:Available: Yes 2024-04-12T21:44:27.242 INFO:teuthology.orchestra.run.smithi123.stdout:Paused: No 2024-04-12T21:44:27.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:44:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:27.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:27 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:44:27.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:44:27.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:27.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:27 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:44:27.870 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-12T21:44:28.887 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:28 smithi123 ceph-mon[24842]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:28.887 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:28 smithi123 ceph-mon[24842]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:28.887 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:28 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:28 smithi177 ceph-mon[26903]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:28 smithi177 ceph-mon[26903]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:28 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (4m) 2m ago 5m 17.5M - ba2b418f427c 82f46b8aa823 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (4m) 65s ago 4m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (3m) 2m ago 5m 42.9M - 8.3.5 dad864ee21e9 6db28cef8fcc 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 6709173bac3d 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (4m) 65s ago 4m 412M - 17.2.0 e1d6a67b021e 632e1f1077ac 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (6m) 2m ago 7m 42.1M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (4m) 65s ago 4m 42.0M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:44:29.830 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (5m) 2m ago 5m 18.0M - 1dbe0e931976 934332c711eb 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (4m) 65s ago 4m 20.7M - 1dbe0e931976 86769ff9a983 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (4m) 2m ago 4m 33.2M 3216M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (3m) 2m ago 3m 34.6M 3216M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (3m) 2m ago 3m 35.3M 3216M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (2m) 2m ago 2m 25.6M 3216M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (2m) 65s ago 2m 35.7M 3984M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (2m) 65s ago 2m 32.8M 3984M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (95s) 65s ago 95s 33.3M 3984M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (71s) 65s ago 70s 12.0M 3984M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (4m) 2m ago 4m 56.2M - 514e6a882f6e 9086f9b4ca30 2024-04-12T21:44:30.430 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-12T21:44:30.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:30 smithi123 ceph-mon[24842]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:30.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:30 smithi177 ceph-mon[26903]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:31.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:31 smithi123 ceph-mon[24842]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:31.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:31 smithi177 ceph-mon[26903]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:32.318 INFO:teuthology.orchestra.run.smithi123.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-12T21:44:32.319 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-04-12T21:44:32.319 INFO:teuthology.orchestra.run.smithi123.stdout:crash 2/2 2m ago 6m * 2024-04-12T21:44:32.319 INFO:teuthology.orchestra.run.smithi123.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-04-12T21:44:32.319 INFO:teuthology.orchestra.run.smithi123.stdout:mgr 2/2 2m ago 6m count:2 2024-04-12T21:44:32.319 INFO:teuthology.orchestra.run.smithi123.stdout:mon 2/2 2m ago 5m smithi123:172.21.15.123=smithi123;smithi177:172.21.15.177=smithi177;count:2 2024-04-12T21:44:32.319 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-04-12T21:44:32.319 INFO:teuthology.orchestra.run.smithi123.stdout:osd 8 2m ago - 2024-04-12T21:44:32.319 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-04-12T21:44:32.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:32 smithi123 ceph-mon[24842]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:32.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:32 smithi177 ceph-mon[26903]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:33.000 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-12T21:44:33.588 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:33 smithi123 ceph-mon[24842]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:33.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:33 smithi177 ceph-mon[26903]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:34.613 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:34 smithi123 ceph-mon[24842]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:34.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:34 smithi177 ceph-mon[26903]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:35.002 INFO:teuthology.orchestra.run.smithi123.stdout:HOST ADDR LABELS STATUS 2024-04-12T21:44:35.002 INFO:teuthology.orchestra.run.smithi123.stdout:smithi123 172.21.15.123 2024-04-12T21:44:35.002 INFO:teuthology.orchestra.run.smithi123.stdout:smithi177 172.21.15.177 2024-04-12T21:44:35.002 INFO:teuthology.orchestra.run.smithi123.stdout:2 hosts in cluster 2024-04-12T21:44:35.638 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-12T21:44:36.887 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:36 smithi123 ceph-mon[24842]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:36.887 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:36 smithi123 ceph-mon[24842]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:37.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:36 smithi177 ceph-mon[26903]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:37.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:36 smithi177 ceph-mon[26903]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:37.532 INFO:teuthology.orchestra.run.smithi123.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-04-12T21:44:37.533 INFO:teuthology.orchestra.run.smithi123.stdout:smithi123 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400YK400BGN 400G LVM detected, locked 2024-04-12T21:44:37.533 INFO:teuthology.orchestra.run.smithi123.stdout:smithi177 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6233007S400BGN 400G LVM detected, locked 2024-04-12T21:44:38.136 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-12T21:44:38.147 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:44:38.148 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-12T21:44:38.176 INFO:teuthology.orchestra.run.smithi123.stderr:+ systemctl stop nfs-server 2024-04-12T21:44:38.184 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi177.front.sepia.ceph.com 2024-04-12T21:44:38.184 DEBUG:teuthology.orchestra.run.smithi177:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-12T21:44:38.217 INFO:teuthology.orchestra.run.smithi177.stderr:+ systemctl stop nfs-server 2024-04-12T21:44:38.225 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T21:44:38.233 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:44:38.234 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-04-12T21:44:38.880 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:38 smithi123 ceph-mon[24842]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:38.880 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:38 smithi123 ceph-mon[24842]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:38 smithi177 ceph-mon[26903]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:38 smithi177 ceph-mon[26903]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:40.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:40 smithi177 ceph-mon[26903]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:40.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:40 smithi123 ceph-mon[24842]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:41.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:41 smithi177 ceph-mon[26903]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:41.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:41 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-12T21:44:41.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:41 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-12T21:44:41.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:41 smithi177 ceph-mon[26903]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T21:44:41.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:41 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-12T21:44:41.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:41 smithi123 ceph-mon[24842]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:44:41.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:41 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-12T21:44:41.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:41 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-12T21:44:41.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:41 smithi123 ceph-mon[24842]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T21:44:41.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:41 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-12T21:44:41.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:41 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[24838]: 2024-04-12T21:44:41.569+0000 7ff2b4c20700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:44:42.277 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-12T21:44:42.286 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-12T21:44:42.286 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T21:44:42.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: pgmap v207: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: fsmap foofs:0 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: Saving service mds.foofs spec with placement count:2 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:42.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:44:42.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:42.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:42.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.poxexz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:44:42.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.poxexz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T21:44:42.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:42.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:42 smithi123 ceph-mon[24842]: Deploying daemon mds.foofs.smithi123.poxexz on smithi123 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: pgmap v207: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: fsmap foofs:0 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: Saving service mds.foofs spec with placement count:2 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:44:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:44:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.poxexz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:44:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.poxexz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T21:44:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:42 smithi177 ceph-mon[26903]: Deploying daemon mds.foofs.smithi123.poxexz on smithi123 2024-04-12T21:44:43.874 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:43 smithi123 ceph-mon[24842]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T21:44:44.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:43 smithi177 ceph-mon[26903]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T21:44:44.700 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:44 smithi123 ceph-mon[24842]: pgmap v211: 65 pgs: 9 creating+peering, 4 creating+activating, 20 active+clean, 32 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:44.701 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:44 smithi123 ceph-mon[24842]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T21:44:45.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:44 smithi177 ceph-mon[26903]: pgmap v211: 65 pgs: 9 creating+peering, 4 creating+activating, 20 active+clean, 32 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:45.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:44 smithi177 ceph-mon[26903]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T21:44:45.387 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:44:45.388 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T21:38:21.360615Z", "last_refresh": "2024-04-12T21:41:57.071141Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T21:38:16.121443Z", "last_refresh": "2024-04-12T21:41:57.071207Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-12T21:38:18.846014Z", "last_refresh": "2024-04-12T21:41:57.071281Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-12T21:44:41.590792Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T21:44:41.585607Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T21:38:14.887058Z", "last_refresh": "2024-04-12T21:41:57.071069Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi177:172.21.15.177=smithi177"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T21:39:02.747478Z", "last_refresh": "2024-04-12T21:41:57.070925Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T21:38:20.083904Z", "last_refresh": "2024-04-12T21:41:57.071344Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-12T21:41:57.071488Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T21:38:17.660968Z", "last_refresh": "2024-04-12T21:41:57.071407Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T21:44:46.116 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-12T21:44:46.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:46 smithi177 ceph-mon[26903]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:46.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:46 smithi177 ceph-mon[26903]: pgmap v213: 65 pgs: 9 creating+peering, 4 creating+activating, 48 active+clean, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:46.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:46 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:46.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:46 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi177.hqctzz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:44:46.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:46 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi177.hqctzz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T21:44:46.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:46 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:46.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:46 smithi123 ceph-mon[24842]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:46.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:46 smithi123 ceph-mon[24842]: pgmap v213: 65 pgs: 9 creating+peering, 4 creating+activating, 48 active+clean, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:46.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:46 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:46.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:46 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi177.hqctzz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:44:46.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:46 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi177.hqctzz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T21:44:46.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:46 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:47.117 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: Deploying daemon mds.foofs.smithi177.hqctzz on smithi177 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: mds.? [v2:172.21.15.123:6834/1104424315,v1:172.21.15.123:6835/1104424315] up:boot 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: daemon mds.foofs.smithi123.poxexz assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: Cluster is now healthy 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: fsmap foofs:0 1 up:standby 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:creating} 2024-04-12T21:44:47.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:47 smithi123 ceph-mon[24842]: daemon mds.foofs.smithi123.poxexz is now active in filesystem foofs as rank 0 2024-04-12T21:44:47.923 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: Deploying daemon mds.foofs.smithi177.hqctzz on smithi177 2024-04-12T21:44:47.923 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: mds.? [v2:172.21.15.123:6834/1104424315,v1:172.21.15.123:6835/1104424315] up:boot 2024-04-12T21:44:47.923 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: daemon mds.foofs.smithi123.poxexz assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-12T21:44:47.923 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T21:44:47.923 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T21:44:47.923 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: Cluster is now healthy 2024-04-12T21:44:47.923 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: fsmap foofs:0 1 up:standby 2024-04-12T21:44:47.923 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:44:47.924 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:creating} 2024-04-12T21:44:47.924 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:47 smithi177 ceph-mon[26903]: daemon mds.foofs.smithi123.poxexz is now active in filesystem foofs as rank 0 2024-04-12T21:44:48.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:48 smithi123 ceph-mon[24842]: pgmap v214: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:48.823 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:48 smithi123 ceph-mon[24842]: mds.? [v2:172.21.15.123:6834/1104424315,v1:172.21.15.123:6835/1104424315] up:active 2024-04-12T21:44:48.823 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:48 smithi123 ceph-mon[24842]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 2024-04-12T21:44:48.860 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:48 smithi177 ceph-mon[26903]: pgmap v214: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:44:48.860 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:48 smithi177 ceph-mon[26903]: mds.? [v2:172.21.15.123:6834/1104424315,v1:172.21.15.123:6835/1104424315] up:active 2024-04-12T21:44:48.860 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:48 smithi177 ceph-mon[26903]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 2024-04-12T21:44:49.283 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:44:49.283 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T21:38:21.360615Z", "last_refresh": "2024-04-12T21:41:57.071141Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T21:38:16.121443Z", "last_refresh": "2024-04-12T21:41:57.071207Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-12T21:38:18.846014Z", "last_refresh": "2024-04-12T21:41:57.071281Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-12T21:44:41.590792Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T21:44:41.585607Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T21:38:14.887058Z", "last_refresh": "2024-04-12T21:41:57.071069Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi177:172.21.15.177=smithi177"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T21:39:02.747478Z", "last_refresh": "2024-04-12T21:41:57.070925Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T21:38:20.083904Z", "last_refresh": "2024-04-12T21:41:57.071344Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-12T21:41:57.071488Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T21:38:17.660968Z", "last_refresh": "2024-04-12T21:41:57.071407Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T21:44:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:44:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:44:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: mds.? [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] up:boot 2024-04-12T21:44:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:44:50.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:44:50.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:49 smithi123 ceph-mon[24842]: pgmap v215: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-04-12T21:44:50.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:44:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:44:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: mds.? [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] up:boot 2024-04-12T21:44:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:44:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:44:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:49 smithi177 ceph-mon[26903]: pgmap v215: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-04-12T21:44:50.349 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-12T21:44:51.350 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T21:44:52.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:51 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:52.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:51 smithi177 ceph-mon[26903]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-12T21:44:52.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:51 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:52.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:51 smithi123 ceph-mon[24842]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-12T21:44:53.975 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:44:53.975 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T21:38:21.360615Z", "last_refresh": "2024-04-12T21:41:57.071141Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T21:38:16.121443Z", "last_refresh": "2024-04-12T21:41:57.071207Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-12T21:38:18.846014Z", "last_refresh": "2024-04-12T21:41:57.071281Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-12T21:44:41.590792Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T21:44:41.585607Z", "last_refresh": "2024-04-12T21:44:50.912654Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T21:38:14.887058Z", "last_refresh": "2024-04-12T21:41:57.071069Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi177:172.21.15.177=smithi177"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T21:39:02.747478Z", "last_refresh": "2024-04-12T21:41:57.070925Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T21:38:20.083904Z", "last_refresh": "2024-04-12T21:41:57.071344Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-12T21:41:57.071488Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T21:38:17.660968Z", "last_refresh": "2024-04-12T21:41:57.071407Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T21:44:54.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:54 smithi123 ceph-mon[24842]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-12T21:44:54.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:54 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:54.933 INFO:tasks.cephadm:mds.foofs has 1/2 2024-04-12T21:44:55.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:54 smithi177 ceph-mon[26903]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-12T21:44:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:54 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:55.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:55 smithi123 ceph-mon[24842]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:55.934 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T21:44:56.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:55 smithi177 ceph-mon[26903]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:44:56.901 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:56 smithi123 ceph-mon[24842]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T21:44:56.902 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:56.902 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:56.902 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:44:56.902 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:56.902 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:44:56.902 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:56 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:57.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:56 smithi177 ceph-mon[26903]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T21:44:57.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:44:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:44:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:44:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:56 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:44:57.893 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:44:57.893 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T21:38:21.360615Z", "last_refresh": "2024-04-12T21:44:56.097299Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T21:38:16.121443Z", "last_refresh": "2024-04-12T21:44:50.912024Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-12T21:38:18.846014Z", "last_refresh": "2024-04-12T21:44:56.097427Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-12T21:44:41.590792Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T21:44:41.585607Z", "last_refresh": "2024-04-12T21:44:50.912654Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T21:38:14.887058Z", "last_refresh": "2024-04-12T21:44:50.912183Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi177:172.21.15.177=smithi177"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T21:39:02.747478Z", "last_refresh": "2024-04-12T21:44:50.912299Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T21:38:20.083904Z", "last_refresh": "2024-04-12T21:44:50.912363Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-12T21:44:50.912422Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T21:38:17.660968Z", "last_refresh": "2024-04-12T21:44:56.097549Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T21:44:58.696 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-12T21:44:58.696 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T21:44:58.705 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:44:58.706 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-12T21:44:58.713 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:58 smithi123 ceph-mon[24842]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T21:44:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:58 smithi177 ceph-mon[26903]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T21:44:59.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:44:59 smithi123 ceph-mon[24842]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:45:00.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:44:59 smithi177 ceph-mon[26903]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T21:45:00.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:00 smithi123 ceph-mon[24842]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T21:45:01.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:00 smithi177 ceph-mon[26903]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T21:45:01.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:01 smithi123 ceph-mon[24842]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:01.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:01 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-12T21:45:02.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:01 smithi177 ceph-mon[26903]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:02.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:01 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-12T21:45:02.609 INFO:teuthology.orchestra.run.smithi123.stdout:NFS Cluster Created Successfully 2024-04-12T21:45:02.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:02 smithi123 ceph-mon[24842]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 2 op/s 2024-04-12T21:45:02.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:02 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-12T21:45:02.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:02 smithi123 ceph-mon[24842]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T21:45:02.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:02 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-12T21:45:03.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:02 smithi177 ceph-mon[26903]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 2 op/s 2024-04-12T21:45:03.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:02 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-12T21:45:03.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:02 smithi177 ceph-mon[26903]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T21:45:03.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:02 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-12T21:45:03.389 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: Saving service nfs.foo spec with placement count:2 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: Creating key for client.nfs.foo.0.0.smithi123.cfxrdw 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T21:45:03.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-12T21:45:03.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:03 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: Saving service nfs.foo spec with placement count:2 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: Creating key for client.nfs.foo.0.0.smithi123.cfxrdw 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-12T21:45:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:03 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:04.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:04 smithi123 ceph-mon[24842]: Rados config object exists: conf-nfs.foo 2024-04-12T21:45:04.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:04 smithi123 ceph-mon[24842]: Creating key for client.nfs.foo.0.0.smithi123.cfxrdw-rgw 2024-04-12T21:45:04.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:04 smithi123 ceph-mon[24842]: Deploying daemon nfs.foo.0.0.smithi123.cfxrdw on smithi123 2024-04-12T21:45:04.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:04 smithi123 ceph-mon[24842]: pgmap v224: 97 pgs: 11 creating+peering, 11 unknown, 75 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:45:04.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:04 smithi123 ceph-mon[24842]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T21:45:04.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:04 smithi123 ceph-mon[24842]: mgrmap e20: smithi123.psjrfs(active, since 5m), standbys: smithi177.whybza 2024-04-12T21:45:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:04 smithi177 ceph-mon[26903]: Rados config object exists: conf-nfs.foo 2024-04-12T21:45:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:04 smithi177 ceph-mon[26903]: Creating key for client.nfs.foo.0.0.smithi123.cfxrdw-rgw 2024-04-12T21:45:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:04 smithi177 ceph-mon[26903]: Deploying daemon nfs.foo.0.0.smithi123.cfxrdw on smithi123 2024-04-12T21:45:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:04 smithi177 ceph-mon[26903]: pgmap v224: 97 pgs: 11 creating+peering, 11 unknown, 75 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-12T21:45:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:04 smithi177 ceph-mon[26903]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T21:45:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:04 smithi177 ceph-mon[26903]: mgrmap e20: smithi123.psjrfs(active, since 5m), standbys: smithi177.whybza 2024-04-12T21:45:06.174 INFO:teuthology.orchestra.run.smithi123.stderr:Invalid command: Unexpected argument '--clusterid' 2024-04-12T21:45:06.174 INFO:teuthology.orchestra.run.smithi123.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-04-12T21:45:06.174 INFO:teuthology.orchestra.run.smithi123.stderr:Error EINVAL: invalid command 2024-04-12T21:45:06.507 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:45:06.507 INFO:teuthology.orchestra.run.smithi123.stdout: "bind": "/fake", 2024-04-12T21:45:06.507 INFO:teuthology.orchestra.run.smithi123.stdout: "fs": "foofs", 2024-04-12T21:45:06.507 INFO:teuthology.orchestra.run.smithi123.stdout: "path": "/", 2024-04-12T21:45:06.507 INFO:teuthology.orchestra.run.smithi123.stdout: "cluster": "foo", 2024-04-12T21:45:06.507 INFO:teuthology.orchestra.run.smithi123.stdout: "mode": "RW" 2024-04-12T21:45:06.507 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:45:06.820 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:06 smithi123 ceph-mon[24842]: pgmap v226: 97 pgs: 11 creating+peering, 1 unknown, 85 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-12T21:45:06.821 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-12T21:45:06.821 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-12T21:45:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:06 smithi177 ceph-mon[26903]: pgmap v226: 97 pgs: 11 creating+peering, 1 unknown, 85 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-12T21:45:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-12T21:45:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-12T21:45:07.218 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-12T21:45:07.820 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:07 smithi123 ceph-mon[24842]: from='client.14524 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:08.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:07 smithi177 ceph-mon[26903]: from='client.14524 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:08.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:08 smithi123 ceph-mon[24842]: pgmap v227: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-12T21:45:08.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:08 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:08.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:08 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T21:45:08.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:08 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-12T21:45:08.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:08 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T21:45:08.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:08 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T21:45:08.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:08 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:09.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:08 smithi177 ceph-mon[26903]: pgmap v227: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-12T21:45:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:08 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:08 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T21:45:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:08 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-12T21:45:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:08 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T21:45:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:08 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T21:45:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:08 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:09.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:09 smithi123 ceph-mon[24842]: Creating key for client.nfs.foo.1.0.smithi177.vvnnsx 2024-04-12T21:45:09.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:09 smithi123 ceph-mon[24842]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-12T21:45:10.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:09 smithi177 ceph-mon[26903]: Creating key for client.nfs.foo.1.0.smithi177.vvnnsx 2024-04-12T21:45:10.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:09 smithi177 ceph-mon[26903]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-12T21:45:10.869 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:10 smithi177 ceph-mon[26903]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 128 B/s rd, 257 B/s wr, 0 op/s 2024-04-12T21:45:10.869 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:10 smithi177 ceph-mon[26903]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:10.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:10 smithi123 ceph-mon[24842]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 128 B/s rd, 257 B/s wr, 0 op/s 2024-04-12T21:45:10.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:10 smithi123 ceph-mon[24842]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:11.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:11 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T21:45:11.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:11 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T21:45:11.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:11 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T21:45:11.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:11 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-12T21:45:11.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:11 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:12.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:11 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T21:45:12.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:11 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T21:45:12.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:11 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T21:45:12.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:11 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-12T21:45:12.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:11 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:12.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:12 smithi123 ceph-mon[24842]: Rados config object exists: conf-nfs.foo 2024-04-12T21:45:12.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:12 smithi123 ceph-mon[24842]: Creating key for client.nfs.foo.1.0.smithi177.vvnnsx-rgw 2024-04-12T21:45:12.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:12 smithi123 ceph-mon[24842]: Deploying daemon nfs.foo.1.0.smithi177.vvnnsx on smithi177 2024-04-12T21:45:12.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:12 smithi123 ceph-mon[24842]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:12.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:12 smithi123 ceph-mon[24842]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-12T21:45:12.945 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:12 smithi177 ceph-mon[26903]: Rados config object exists: conf-nfs.foo 2024-04-12T21:45:12.945 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:12 smithi177 ceph-mon[26903]: Creating key for client.nfs.foo.1.0.smithi177.vvnnsx-rgw 2024-04-12T21:45:12.945 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:12 smithi177 ceph-mon[26903]: Deploying daemon nfs.foo.1.0.smithi177.vvnnsx on smithi177 2024-04-12T21:45:12.945 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:12 smithi177 ceph-mon[26903]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:12.946 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:12 smithi177 ceph-mon[26903]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-12T21:45:13.867 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:13 smithi177 ceph-mon[26903]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:13.867 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:13 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:13.867 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:13 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:45:13.867 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:13 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:13.867 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:13 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:45:13.867 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:13 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:13.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:13 smithi123 ceph-mon[24842]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:13.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:13 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:13.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:13 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:45:13.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:13 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:13.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:13 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:45:13.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:13 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:14.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:14 smithi177 ceph-mon[26903]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-12T21:45:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:14 smithi177 ceph-mon[26903]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:14.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:14 smithi123 ceph-mon[24842]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-12T21:45:14.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:14 smithi123 ceph-mon[24842]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:17.006 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:16 smithi123 ceph-mon[24842]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:17.007 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:16 smithi123 ceph-mon[24842]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.3 KiB/s wr, 6 op/s 2024-04-12T21:45:17.007 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:16 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:17.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:16 smithi177 ceph-mon[26903]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:17.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:16 smithi177 ceph-mon[26903]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.3 KiB/s wr, 6 op/s 2024-04-12T21:45:17.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:16 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:18.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:17 smithi177 ceph-mon[26903]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:18.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:17 smithi123 ceph-mon[24842]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:18.823 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo ?:2049 2/2 3s ago 16s count:2 2024-04-12T21:45:19.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:18 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:18 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:19.487 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-12T21:45:19.498 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:45:19.498 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-12T21:45:19.525 INFO:teuthology.orchestra.run.smithi123.stderr:+ mkdir /mnt/foo 2024-04-12T21:45:19.542 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-12T21:45:19.608 INFO:teuthology.orchestra.run.smithi123.stderr:++ hostname 2024-04-12T21:45:19.608 INFO:teuthology.orchestra.run.smithi123.stderr:+ mount -t nfs smithi123:/fake /mnt/foo -o sync 2024-04-12T21:45:20.011 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-12T21:45:20.078 INFO:teuthology.orchestra.run.smithi123.stderr:+ echo test 2024-04-12T21:45:20.089 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-12T21:45:20.161 INFO:teuthology.orchestra.run.smithi123.stderr:+ sync 2024-04-12T21:45:20.275 INFO:teuthology.run_tasks:Running task parallel... 2024-04-12T21:45:20.285 INFO:teuthology.task.parallel:starting parallel... 2024-04-12T21:45:20.285 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T21:45:20.285 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-12T21:45:20.286 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:45:20.286 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-12T21:45:20.286 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T21:45:20.286 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-12T21:45:20.287 INFO:teuthology.task.exec:Executing custom commands... 2024-04-12T21:45:20.287 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:45:20.288 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-12T21:45:20.383 INFO:teuthology.orchestra.run.smithi123.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-12T21:45:20.383 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:45:20.383 INFO:teuthology.orchestra.run.smithi123.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-12T21:45:20.384 INFO:teuthology.orchestra.run.smithi123.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-12T21:45:20.485 INFO:teuthology.orchestra.run.smithi123.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-12T21:45:20.485 INFO:teuthology.orchestra.run.smithi123.stdout:releasing clients 2024-04-12T21:45:20.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:20 smithi123 ceph-mon[24842]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:20.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:20 smithi123 ceph-mon[24842]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-12T21:45:20.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:20 smithi177 ceph-mon[26903]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:20.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:20 smithi177 ceph-mon[26903]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-12T21:45:21.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 137 29.69 MB/sec warmup 1 sec latency 39.386 ms 2024-04-12T21:45:22.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 268 25.12 MB/sec warmup 2 sec latency 31.703 ms 2024-04-12T21:45:22.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:22 smithi177 ceph-mon[26903]: pgmap v234: 97 pgs: 97 active+clean; 587 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.9 KiB/s wr, 6 op/s 2024-04-12T21:45:22.817 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:22 smithi123 ceph-mon[24842]: pgmap v234: 97 pgs: 97 active+clean; 587 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.9 KiB/s wr, 6 op/s 2024-04-12T21:45:23.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 378 22.45 MB/sec warmup 3 sec latency 79.044 ms 2024-04-12T21:45:23.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-12T21:45:23.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-12T21:45:23.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-12T21:45:23.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-12T21:45:23.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:45:23.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:45:23.948 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-12T21:45:24.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-12T21:45:24.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-12T21:45:24.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-12T21:45:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-12T21:45:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:45:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:45:24.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 482 21.03 MB/sec warmup 4 sec latency 46.865 ms 2024-04-12T21:45:24.626 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:24 smithi123 ceph-mon[24842]: pgmap v235: 97 pgs: 97 active+clean; 21 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.7 MiB/s wr, 35 op/s 2024-04-12T21:45:24.626 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-04-12T21:45:24.626 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-12T21:45:24.626 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-04-12T21:45:24.626 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-12T21:45:24.627 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:24 smithi123 ceph-mon[24842]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T21:45:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:24 smithi177 ceph-mon[26903]: pgmap v235: 97 pgs: 97 active+clean; 21 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.7 MiB/s wr, 35 op/s 2024-04-12T21:45:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-04-12T21:45:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-12T21:45:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-04-12T21:45:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-12T21:45:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:24 smithi177 ceph-mon[26903]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T21:45:25.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 637 20.84 MB/sec warmup 5 sec latency 29.644 ms 2024-04-12T21:45:25.704 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:25 smithi123 ceph-mon[24842]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T21:45:26.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:25 smithi177 ceph-mon[26903]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T21:45:26.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 1010 18.52 MB/sec warmup 6 sec latency 42.822 ms 2024-04-12T21:45:26.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:26 smithi123 ceph-mon[24842]: pgmap v238: 97 pgs: 97 active+clean; 68 MiB data, 108 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 8.7 MiB/s wr, 383 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:27.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:26 smithi177 ceph-mon[26903]: pgmap v238: 97 pgs: 97 active+clean; 68 MiB data, 108 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 8.7 MiB/s wr, 383 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:27.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 1615 18.69 MB/sec warmup 7 sec latency 86.328 ms 2024-04-12T21:45:28.052 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-12T21:45:28.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 2485 17.89 MB/sec warmup 8 sec latency 23.573 ms 2024-04-12T21:45:28.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:28 smithi123 ceph-mon[24842]: pgmap v239: 97 pgs: 97 active+clean; 87 MiB data, 134 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 11 MiB/s wr, 440 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:29.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:28 smithi177 ceph-mon[26903]: pgmap v239: 97 pgs: 97 active+clean; 87 MiB data, 134 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 11 MiB/s wr, 440 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:29.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 3346 19.67 MB/sec warmup 9 sec latency 19.803 ms 2024-04-12T21:45:30.486 INFO:teuthology.orchestra.run.smithi123.stdout: 5 3940 20.03 MB/sec warmup 10 sec latency 25.238 ms 2024-04-12T21:45:30.509 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:30 smithi123 ceph-mon[24842]: pgmap v240: 97 pgs: 97 active+clean; 100 MiB data, 143 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 13 MiB/s wr, 505 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:30.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:30 smithi177 ceph-mon[26903]: pgmap v240: 97 pgs: 97 active+clean; 100 MiB data, 143 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 13 MiB/s wr, 505 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:31.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 4573 18.80 MB/sec warmup 11 sec latency 43.258 ms 2024-04-12T21:45:32.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 5305 18.91 MB/sec warmup 12 sec latency 48.721 ms 2024-04-12T21:45:32.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:32 smithi123 ceph-mon[24842]: pgmap v241: 97 pgs: 97 active+clean; 121 MiB data, 171 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 15 MiB/s wr, 842 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:33.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:32 smithi177 ceph-mon[26903]: pgmap v241: 97 pgs: 97 active+clean; 121 MiB data, 171 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 15 MiB/s wr, 842 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:33.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 5994 18.20 MB/sec warmup 13 sec latency 53.287 ms 2024-04-12T21:45:33.880 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph mgr module enable nfs --force' 2024-04-12T21:45:34.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 6680 19.05 MB/sec warmup 14 sec latency 37.448 ms 2024-04-12T21:45:34.786 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:34 smithi123 ceph-mon[24842]: pgmap v242: 97 pgs: 97 active+clean; 133 MiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 14 MiB/s wr, 864 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:35.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:34 smithi177 ceph-mon[26903]: pgmap v242: 97 pgs: 97 active+clean; 133 MiB data, 235 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 14 MiB/s wr, 864 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:35.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 7527 19.73 MB/sec warmup 15 sec latency 21.584 ms 2024-04-12T21:45:36.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 8439 19.71 MB/sec warmup 16 sec latency 10.361 ms 2024-04-12T21:45:36.795 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:36 smithi123 ceph-mon[24842]: pgmap v243: 97 pgs: 97 active+clean; 142 MiB data, 291 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 14 MiB/s wr, 1.01k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:37.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:36 smithi177 ceph-mon[26903]: pgmap v243: 97 pgs: 97 active+clean; 142 MiB data, 291 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 14 MiB/s wr, 1.01k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T21:45:37.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 9422 19.39 MB/sec warmup 17 sec latency 39.745 ms 2024-04-12T21:45:37.634 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:37 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1028064623' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-12T21:45:37.661 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:37 smithi123 ceph-mon[24842]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-12T21:45:37.662 INFO:teuthology.orchestra.run.smithi123.stderr:module 'nfs' is already enabled 2024-04-12T21:45:38.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:37 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1028064623' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-12T21:45:38.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:37 smithi177 ceph-mon[26903]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-12T21:45:38.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 10384 20.44 MB/sec warmup 18 sec latency 24.706 ms 2024-04-12T21:45:38.651 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:38 smithi123 ceph-mon[24842]: pgmap v244: 97 pgs: 97 active+clean; 142 MiB data, 305 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 11 MiB/s wr, 796 op/s 2024-04-12T21:45:38.651 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:38 smithi123 ceph-mon[24842]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-12T21:45:38.651 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:38 smithi123 ceph-mon[24842]: mgrmap e21: smithi123.psjrfs(active, since 6m), standbys: smithi177.whybza 2024-04-12T21:45:39.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:38 smithi177 ceph-mon[26903]: pgmap v244: 97 pgs: 97 active+clean; 142 MiB data, 305 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 11 MiB/s wr, 796 op/s 2024-04-12T21:45:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:38 smithi177 ceph-mon[26903]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-12T21:45:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:38 smithi177 ceph-mon[26903]: mgrmap e21: smithi123.psjrfs(active, since 6m), standbys: smithi177.whybza 2024-04-12T21:45:39.095 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-12T21:45:39.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 11063 20.60 MB/sec warmup 19 sec latency 14.277 ms 2024-04-12T21:45:40.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 11934 20.26 MB/sec warmup 20 sec latency 10.912 ms 2024-04-12T21:45:40.660 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:40 smithi123 ceph-mon[24842]: pgmap v245: 97 pgs: 97 active+clean; 140 MiB data, 344 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 10 MiB/s wr, 811 op/s 2024-04-12T21:45:40.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:40 smithi177 ceph-mon[26903]: pgmap v245: 97 pgs: 97 active+clean; 140 MiB data, 344 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 10 MiB/s wr, 811 op/s 2024-04-12T21:45:41.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 13039 20.32 MB/sec warmup 21 sec latency 11.898 ms 2024-04-12T21:45:42.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 13985 21.19 MB/sec warmup 22 sec latency 45.998 ms 2024-04-12T21:45:42.682 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:42 smithi123 ceph-mon[24842]: pgmap v246: 97 pgs: 97 active+clean; 150 MiB data, 406 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-04-12T21:45:42.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:42 smithi177 ceph-mon[26903]: pgmap v246: 97 pgs: 97 active+clean; 150 MiB data, 406 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-04-12T21:45:42.932 INFO:teuthology.orchestra.run.smithi123.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T21:45:43.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 14527 21.13 MB/sec warmup 23 sec latency 23.705 ms 2024-04-12T21:45:43.947 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:43.947 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T21:45:43.947 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:43.947 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:45:43.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:43.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:45:43.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:43.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:43.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T21:45:43.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:43 smithi123 ceph-mon[24842]: pgmap v247: 97 pgs: 97 active+clean; 160 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.10k op/s 2024-04-12T21:45:44.053 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-12T21:45:44.053 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T21:45:44.054 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-12T21:45:44.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:44.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T21:45:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:45:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:45:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:45:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:45:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T21:45:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:43 smithi177 ceph-mon[26903]: pgmap v247: 97 pgs: 97 active+clean; 160 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.10k op/s 2024-04-12T21:45:44.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 15091 20.48 MB/sec warmup 24 sec latency 28.093 ms 2024-04-12T21:45:45.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 16103 20.58 MB/sec warmup 25 sec latency 13.097 ms 2024-04-12T21:45:46.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 17119 21.07 MB/sec warmup 26 sec latency 15.754 ms 2024-04-12T21:45:46.709 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:46 smithi123 ceph-mon[24842]: pgmap v248: 97 pgs: 97 active+clean; 166 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-04-12T21:45:47.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:46 smithi177 ceph-mon[26903]: pgmap v248: 97 pgs: 97 active+clean; 166 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-04-12T21:45:47.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 17954 21.46 MB/sec warmup 27 sec latency 28.411 ms 2024-04-12T21:45:47.973 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:45:48.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 18426 20.86 MB/sec warmup 28 sec latency 26.195 ms 2024-04-12T21:45:48.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:48 smithi123 ceph-mon[24842]: pgmap v249: 97 pgs: 97 active+clean; 170 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-12T21:45:49.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:48 smithi177 ceph-mon[26903]: pgmap v249: 97 pgs: 97 active+clean; 170 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (5m) 30s ago 6m 24.3M - ba2b418f427c 82f46b8aa823 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (6m) 30s ago 6m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (6m) 33s ago 6m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (5m) 30s ago 6m 52.5M - 8.3.5 dad864ee21e9 6db28cef8fcc 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (63s) 30s ago 62s 14.9M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (60s) 33s ago 60s 12.2M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:9283 running (8m) 30s ago 8m 460M - 17.2.0 e1d6a67b021e 6709173bac3d 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (6m) 33s ago 6m 412M - 17.2.0 e1d6a67b021e 632e1f1077ac 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (8m) 30s ago 8m 47.6M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (6m) 33s ago 6m 37.2M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:45:49.032 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (41s) 30s ago 41s 46.8M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (36s) 33s ago 35s 46.6M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (6m) 30s ago 6m 20.0M - 1dbe0e931976 934332c711eb 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (6m) 33s ago 6m 17.9M - 1dbe0e931976 86769ff9a983 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (5m) 30s ago 5m 42.8M 3216M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (5m) 30s ago 5m 43.0M 3216M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (4m) 30s ago 4m 44.2M 3216M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (4m) 30s ago 4m 40.0M 3216M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (3m) 33s ago 3m 42.3M 3984M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (3m) 33s ago 3m 38.6M 3984M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (2m) 33s ago 2m 42.0M 3984M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (2m) 33s ago 2m 40.5M 3984M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:45:49.033 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (5m) 30s ago 5m 62.8M - 514e6a882f6e 9086f9b4ca30 2024-04-12T21:45:49.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 18974 20.55 MB/sec warmup 29 sec latency 27.439 ms 2024-04-12T21:45:49.663 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:45:49.663 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:45:49.664 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:45:49.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:49 smithi123 ceph-mon[24842]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:49.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:49 smithi123 ceph-mon[24842]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:50.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:49 smithi177 ceph-mon[26903]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:50.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:49 smithi177 ceph-mon[26903]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:50.181 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:45:50.182 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", 2024-04-12T21:45:50.182 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:45:50.182 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-04-12T21:45:50.182 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "", 2024-04-12T21:45:50.182 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a image" 2024-04-12T21:45:50.182 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:45:50.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 19881 20.41 MB/sec warmup 30 sec latency 14.584 ms 2024-04-12T21:45:50.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:50 smithi123 ceph-mon[24842]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:50.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:50 smithi123 ceph-mon[24842]: pgmap v250: 97 pgs: 97 active+clean; 175 MiB data, 562 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.07k op/s 2024-04-12T21:45:50.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:50 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2936422286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:45:50.820 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-04-12T21:45:51.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:50 smithi177 ceph-mon[26903]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:51.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:50 smithi177 ceph-mon[26903]: pgmap v250: 97 pgs: 97 active+clean; 175 MiB data, 562 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.07k op/s 2024-04-12T21:45:51.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:50 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2936422286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:45:51.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 20641 20.59 MB/sec warmup 31 sec latency 27.883 ms 2024-04-12T21:45:51.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:51 smithi123 ceph-mon[24842]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:51.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:51 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2052439062' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:45:52.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:51 smithi177 ceph-mon[26903]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:45:52.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:51 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2052439062' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:45:52.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 21602 21.12 MB/sec warmup 32 sec latency 14.712 ms 2024-04-12T21:45:52.795 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:52 smithi123 ceph-mon[24842]: pgmap v251: 97 pgs: 97 active+clean; 181 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 13 MiB/s wr, 1.17k op/s 2024-04-12T21:45:53.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:52 smithi177 ceph-mon[26903]: pgmap v251: 97 pgs: 97 active+clean; 181 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 13 MiB/s wr, 1.17k op/s 2024-04-12T21:45:53.487 INFO:teuthology.orchestra.run.smithi123.stdout: 5 22207 20.68 MB/sec warmup 33 sec latency 23.635 ms 2024-04-12T21:45:54.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 22774 20.54 MB/sec warmup 34 sec latency 80.284 ms 2024-04-12T21:45:54.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:54 smithi123 ceph-mon[24842]: pgmap v252: 97 pgs: 97 active+clean; 187 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-04-12T21:45:55.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:54 smithi177 ceph-mon[26903]: pgmap v252: 97 pgs: 97 active+clean; 187 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-04-12T21:45:55.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 23460 20.24 MB/sec warmup 35 sec latency 30.749 ms 2024-04-12T21:45:56.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 23900 20.03 MB/sec warmup 36 sec latency 34.166 ms 2024-04-12T21:45:56.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:56 smithi123 ceph-mon[24842]: pgmap v253: 97 pgs: 97 active+clean; 188 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 997 op/s 2024-04-12T21:45:57.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:56 smithi177 ceph-mon[26903]: pgmap v253: 97 pgs: 97 active+clean; 188 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 997 op/s 2024-04-12T21:45:57.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 24529 20.25 MB/sec warmup 37 sec latency 23.642 ms 2024-04-12T21:45:58.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 25077 20.28 MB/sec warmup 38 sec latency 35.037 ms 2024-04-12T21:45:58.845 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:45:58 smithi123 ceph-mon[24842]: pgmap v254: 97 pgs: 97 active+clean; 189 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 928 op/s 2024-04-12T21:45:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:45:58 smithi177 ceph-mon[26903]: pgmap v254: 97 pgs: 97 active+clean; 189 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 928 op/s 2024-04-12T21:45:59.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 25658 19.92 MB/sec warmup 39 sec latency 20.459 ms 2024-04-12T21:46:00.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 26452 20.00 MB/sec warmup 40 sec latency 25.377 ms 2024-04-12T21:46:00.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:00 smithi123 ceph-mon[24842]: pgmap v255: 97 pgs: 97 active+clean; 188 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 820 op/s 2024-04-12T21:46:00.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:00 smithi177 ceph-mon[26903]: pgmap v255: 97 pgs: 97 active+clean; 188 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 820 op/s 2024-04-12T21:46:01.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 27297 19.80 MB/sec warmup 41 sec latency 20.865 ms 2024-04-12T21:46:02.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 28002 20.03 MB/sec warmup 42 sec latency 26.344 ms 2024-04-12T21:46:02.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:02 smithi123 ceph-mon[24842]: pgmap v256: 97 pgs: 97 active+clean; 197 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 941 op/s 2024-04-12T21:46:03.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:02 smithi177 ceph-mon[26903]: pgmap v256: 97 pgs: 97 active+clean; 197 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 941 op/s 2024-04-12T21:46:03.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 28699 20.15 MB/sec warmup 43 sec latency 22.778 ms 2024-04-12T21:46:04.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 29409 19.96 MB/sec warmup 44 sec latency 20.831 ms 2024-04-12T21:46:04.634 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:04 smithi123 ceph-mon[24842]: pgmap v257: 97 pgs: 97 active+clean; 209 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 1.05k op/s 2024-04-12T21:46:05.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:04 smithi177 ceph-mon[26903]: pgmap v257: 97 pgs: 97 active+clean; 209 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 1.05k op/s 2024-04-12T21:46:05.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 30348 19.97 MB/sec warmup 45 sec latency 13.637 ms 2024-04-12T21:46:06.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 31210 20.05 MB/sec warmup 46 sec latency 19.479 ms 2024-04-12T21:46:06.642 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:06 smithi123 ceph-mon[24842]: pgmap v258: 97 pgs: 97 active+clean; 216 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 973 op/s 2024-04-12T21:46:07.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:06 smithi177 ceph-mon[26903]: pgmap v258: 97 pgs: 97 active+clean; 216 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 973 op/s 2024-04-12T21:46:07.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 32031 20.31 MB/sec warmup 47 sec latency 25.199 ms 2024-04-12T21:46:08.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 32608 20.16 MB/sec warmup 48 sec latency 28.883 ms 2024-04-12T21:46:08.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:08 smithi123 ceph-mon[24842]: pgmap v259: 97 pgs: 97 active+clean; 217 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.2 MiB/s wr, 862 op/s 2024-04-12T21:46:09.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:08 smithi177 ceph-mon[26903]: pgmap v259: 97 pgs: 97 active+clean; 217 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.2 MiB/s wr, 862 op/s 2024-04-12T21:46:09.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 33401 20.11 MB/sec warmup 49 sec latency 14.945 ms 2024-04-12T21:46:10.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 34285 20.05 MB/sec warmup 50 sec latency 19.977 ms 2024-04-12T21:46:10.508 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:10 smithi123 ceph-mon[24842]: pgmap v260: 97 pgs: 97 active+clean; 220 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.4 MiB/s wr, 805 op/s 2024-04-12T21:46:10.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:10 smithi177 ceph-mon[26903]: pgmap v260: 97 pgs: 97 active+clean; 220 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.4 MiB/s wr, 805 op/s 2024-04-12T21:46:11.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 35220 20.33 MB/sec warmup 51 sec latency 11.129 ms 2024-04-12T21:46:12.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 35914 20.38 MB/sec warmup 52 sec latency 18.468 ms 2024-04-12T21:46:12.777 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:12 smithi123 ceph-mon[24842]: pgmap v261: 97 pgs: 97 active+clean; 225 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-04-12T21:46:12.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:12 smithi177 ceph-mon[26903]: pgmap v261: 97 pgs: 97 active+clean; 225 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-04-12T21:46:13.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 36587 20.21 MB/sec warmup 53 sec latency 16.134 ms 2024-04-12T21:46:14.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 37267 20.06 MB/sec warmup 54 sec latency 21.637 ms 2024-04-12T21:46:14.791 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:14 smithi123 ceph-mon[24842]: pgmap v262: 97 pgs: 97 active+clean; 227 MiB data, 1012 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-12T21:46:15.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:14 smithi177 ceph-mon[26903]: pgmap v262: 97 pgs: 97 active+clean; 227 MiB data, 1012 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-12T21:46:15.488 INFO:teuthology.orchestra.run.smithi123.stdout: 5 37818 19.96 MB/sec warmup 55 sec latency 32.908 ms 2024-04-12T21:46:16.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 38305 19.87 MB/sec warmup 56 sec latency 52.994 ms 2024-04-12T21:46:16.815 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:16 smithi123 ceph-mon[24842]: pgmap v263: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-04-12T21:46:17.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:16 smithi177 ceph-mon[26903]: pgmap v263: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-04-12T21:46:17.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 38957 20.02 MB/sec warmup 57 sec latency 21.359 ms 2024-04-12T21:46:18.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 39435 19.91 MB/sec warmup 58 sec latency 22.412 ms 2024-04-12T21:46:18.824 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:18 smithi123 ceph-mon[24842]: pgmap v264: 97 pgs: 97 active+clean; 236 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 892 op/s 2024-04-12T21:46:19.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:18 smithi177 ceph-mon[26903]: pgmap v264: 97 pgs: 97 active+clean; 236 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 892 op/s 2024-04-12T21:46:19.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 40012 19.73 MB/sec warmup 59 sec latency 33.003 ms 2024-04-12T21:46:20.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 40605 19.63 MB/sec warmup 60 sec latency 47.772 ms 2024-04-12T21:46:20.616 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:20 smithi123 ceph-mon[24842]: pgmap v265: 97 pgs: 97 active+clean; 234 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 879 op/s 2024-04-12T21:46:20.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:20 smithi177 ceph-mon[26903]: pgmap v265: 97 pgs: 97 active+clean; 234 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 879 op/s 2024-04-12T21:46:21.399 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:46:21.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 41204 19.52 MB/sec warmup 61 sec latency 88.150 ms 2024-04-12T21:46:22.418 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:46:22.418 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (6m) 64s ago 7m 24.3M - ba2b418f427c 82f46b8aa823 2024-04-12T21:46:22.418 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (7m) 64s ago 7m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (6m) 66s ago 6m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (5m) 64s ago 7m 52.5M - 8.3.5 dad864ee21e9 6db28cef8fcc 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (97s) 64s ago 96s 14.9M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (93s) 66s ago 93s 12.2M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:9283 running (8m) 64s ago 8m 460M - 17.2.0 e1d6a67b021e 6709173bac3d 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (6m) 66s ago 6m 412M - 17.2.0 e1d6a67b021e 632e1f1077ac 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (8m) 64s ago 8m 47.6M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (6m) 66s ago 6m 37.2M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (75s) 64s ago 74s 46.8M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (69s) 66s ago 69s 46.6M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 64s ago 7m 20.0M - 1dbe0e931976 934332c711eb 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (6m) 66s ago 6m 17.9M - 1dbe0e931976 86769ff9a983 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (6m) 64s ago 6m 42.8M 3216M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:46:22.419 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (5m) 64s ago 5m 43.0M 3216M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:46:22.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (5m) 64s ago 5m 44.2M 3216M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:46:22.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (4m) 64s ago 4m 40.0M 3216M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:46:22.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (4m) 66s ago 4m 42.3M 3984M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:46:22.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (3m) 66s ago 3m 38.6M 3984M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:46:22.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (3m) 66s ago 3m 42.0M 3984M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:46:22.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (3m) 66s ago 3m 40.5M 3984M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:46:22.420 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (6m) 64s ago 6m 62.8M - 514e6a882f6e 9086f9b4ca30 2024-04-12T21:46:22.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 41788 19.43 MB/sec warmup 62 sec latency 24.943 ms 2024-04-12T21:46:22.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:22 smithi123 ceph-mon[24842]: from='client.24377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:22.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:22 smithi123 ceph-mon[24842]: pgmap v266: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-12T21:46:22.626 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:46:22.626 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:46:22.626 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:46:22.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:22 smithi177 ceph-mon[26903]: from='client.24377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:22.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:22 smithi177 ceph-mon[26903]: pgmap v266: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-12T21:46:22.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:46:22.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:46:22.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:46:23.055 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:46:23.055 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:46:23.055 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:46:23.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:46:23.055 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:46:23.055 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:46:23.055 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:46:23.055 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:46:23.056 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:46:23.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 42355 19.51 MB/sec warmup 63 sec latency 30.027 ms 2024-04-12T21:46:23.491 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:46:23.491 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:46:23.491 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:46:23.491 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-04-12T21:46:23.491 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "0/23 daemons upgraded", 2024-04-12T21:46:23.492 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec image on host smithi177" 2024-04-12T21:46:23.492 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:46:23.652 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:23 smithi123 ceph-mon[24842]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:23.652 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:23 smithi123 ceph-mon[24842]: from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:23.652 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:23 smithi123 ceph-mon[24842]: Upgrade: Target is version 19.0.0-2053-gb277feb2 (unknown) 2024-04-12T21:46:23.652 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:23 smithi123 ceph-mon[24842]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec'] 2024-04-12T21:46:23.652 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:23 smithi123 ceph-mon[24842]: Upgrade: Need to upgrade myself (mgr.smithi123.psjrfs) 2024-04-12T21:46:23.652 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:23 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3785817625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:46:23.653 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:46:23.653 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:23 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:46:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:23 smithi177 ceph-mon[26903]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:23 smithi177 ceph-mon[26903]: from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:23 smithi177 ceph-mon[26903]: Upgrade: Target is version 19.0.0-2053-gb277feb2 (unknown) 2024-04-12T21:46:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:23 smithi177 ceph-mon[26903]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec'] 2024-04-12T21:46:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:23 smithi177 ceph-mon[26903]: Upgrade: Need to upgrade myself (mgr.smithi123.psjrfs) 2024-04-12T21:46:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:23 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3785817625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:46:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:46:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:23 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:46:24.053 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-04-12T21:46:24.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 42974 19.54 MB/sec warmup 64 sec latency 27.533 ms 2024-04-12T21:46:24.657 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:24 smithi123 ceph-mon[24842]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec on smithi177 2024-04-12T21:46:24.657 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:24 smithi123 ceph-mon[24842]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:24.658 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:24 smithi123 ceph-mon[24842]: pgmap v267: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.5 MiB/s wr, 895 op/s 2024-04-12T21:46:24.658 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:24 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/4254693193' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:46:25.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:24 smithi177 ceph-mon[26903]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec on smithi177 2024-04-12T21:46:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:24 smithi177 ceph-mon[26903]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:24 smithi177 ceph-mon[26903]: pgmap v267: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.5 MiB/s wr, 895 op/s 2024-04-12T21:46:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:24 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/4254693193' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:46:25.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 43414 19.36 MB/sec warmup 65 sec latency 43.164 ms 2024-04-12T21:46:26.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 43987 19.21 MB/sec warmup 66 sec latency 31.532 ms 2024-04-12T21:46:26.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:26 smithi123 ceph-mon[24842]: pgmap v268: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.9 MiB/s wr, 787 op/s 2024-04-12T21:46:27.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:26 smithi177 ceph-mon[26903]: pgmap v268: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.9 MiB/s wr, 787 op/s 2024-04-12T21:46:27.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 44603 19.14 MB/sec warmup 67 sec latency 39.333 ms 2024-04-12T21:46:28.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 45406 19.22 MB/sec warmup 68 sec latency 25.768 ms 2024-04-12T21:46:28.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:28 smithi123 ceph-mon[24842]: pgmap v269: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.3 MiB/s wr, 609 op/s 2024-04-12T21:46:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:28 smithi177 ceph-mon[26903]: pgmap v269: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.3 MiB/s wr, 609 op/s 2024-04-12T21:46:29.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 46346 19.44 MB/sec warmup 69 sec latency 19.138 ms 2024-04-12T21:46:30.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 47166 19.38 MB/sec warmup 70 sec latency 24.427 ms 2024-04-12T21:46:30.508 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:30 smithi123 ceph-mon[24842]: pgmap v270: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.6 MiB/s wr, 677 op/s 2024-04-12T21:46:30.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:30 smithi177 ceph-mon[26903]: pgmap v270: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.6 MiB/s wr, 677 op/s 2024-04-12T21:46:31.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 48098 19.40 MB/sec warmup 71 sec latency 18.255 ms 2024-04-12T21:46:32.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 49007 19.50 MB/sec warmup 72 sec latency 36.448 ms 2024-04-12T21:46:32.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:32 smithi123 ceph-mon[24842]: pgmap v271: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 870 op/s 2024-04-12T21:46:33.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:32 smithi177 ceph-mon[26903]: pgmap v271: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 870 op/s 2024-04-12T21:46:33.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 49922 19.72 MB/sec warmup 73 sec latency 62.838 ms 2024-04-12T21:46:34.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 50451 19.55 MB/sec warmup 74 sec latency 24.785 ms 2024-04-12T21:46:34.789 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:34 smithi123 ceph-mon[24842]: pgmap v272: 97 pgs: 97 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-04-12T21:46:35.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:34 smithi177 ceph-mon[26903]: pgmap v272: 97 pgs: 97 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-04-12T21:46:35.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 51150 19.47 MB/sec warmup 75 sec latency 19.977 ms 2024-04-12T21:46:36.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 51870 19.45 MB/sec warmup 76 sec latency 39.451 ms 2024-04-12T21:46:36.810 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:36 smithi123 ceph-mon[24842]: pgmap v273: 97 pgs: 97 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-12T21:46:37.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:36 smithi177 ceph-mon[26903]: pgmap v273: 97 pgs: 97 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-04-12T21:46:37.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 52455 19.42 MB/sec warmup 77 sec latency 52.630 ms 2024-04-12T21:46:38.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 53220 19.55 MB/sec warmup 78 sec latency 30.994 ms 2024-04-12T21:46:38.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:38 smithi123 ceph-mon[24842]: pgmap v274: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.08k op/s 2024-04-12T21:46:39.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:38 smithi177 ceph-mon[26903]: pgmap v274: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 1.08k op/s 2024-04-12T21:46:39.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 53770 19.49 MB/sec warmup 79 sec latency 27.791 ms 2024-04-12T21:46:40.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 54335 19.36 MB/sec warmup 80 sec latency 25.248 ms 2024-04-12T21:46:40.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:40 smithi123 ceph-mon[24842]: pgmap v275: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-12T21:46:40.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:40 smithi177 ceph-mon[26903]: pgmap v275: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-12T21:46:41.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 54994 19.28 MB/sec warmup 81 sec latency 34.455 ms 2024-04-12T21:46:42.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 55605 19.24 MB/sec warmup 82 sec latency 64.949 ms 2024-04-12T21:46:42.689 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:42 smithi123 ceph-mon[24842]: pgmap v276: 97 pgs: 97 active+clean; 290 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2024-04-12T21:46:43.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:42 smithi177 ceph-mon[26903]: pgmap v276: 97 pgs: 97 active+clean; 290 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.13k op/s 2024-04-12T21:46:43.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 56272 19.27 MB/sec warmup 83 sec latency 45.705 ms 2024-04-12T21:46:44.489 INFO:teuthology.orchestra.run.smithi123.stdout: 5 56828 19.32 MB/sec warmup 84 sec latency 60.761 ms 2024-04-12T21:46:44.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:44 smithi123 ceph-mon[24842]: pgmap v277: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-04-12T21:46:45.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:44 smithi177 ceph-mon[26903]: pgmap v277: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-04-12T21:46:45.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 57354 19.24 MB/sec warmup 85 sec latency 23.048 ms 2024-04-12T21:46:46.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 58078 19.17 MB/sec warmup 86 sec latency 25.449 ms 2024-04-12T21:46:46.722 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:46 smithi123 ceph-mon[24842]: pgmap v278: 97 pgs: 97 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 920 op/s 2024-04-12T21:46:47.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:46 smithi177 ceph-mon[26903]: pgmap v278: 97 pgs: 97 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 920 op/s 2024-04-12T21:46:47.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 58987 19.20 MB/sec warmup 87 sec latency 51.881 ms 2024-04-12T21:46:48.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 59778 19.26 MB/sec warmup 88 sec latency 31.481 ms 2024-04-12T21:46:48.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:48 smithi123 ceph-mon[24842]: pgmap v279: 97 pgs: 97 active+clean; 303 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 8.7 MiB/s wr, 842 op/s 2024-04-12T21:46:49.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:48 smithi177 ceph-mon[26903]: pgmap v279: 97 pgs: 97 active+clean; 303 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 8.7 MiB/s wr, 842 op/s 2024-04-12T21:46:49.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 60514 19.37 MB/sec warmup 89 sec latency 25.826 ms 2024-04-12T21:46:50.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 61145 19.27 MB/sec warmup 90 sec latency 26.884 ms 2024-04-12T21:46:50.501 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:50 smithi123 ceph-mon[24842]: pgmap v280: 97 pgs: 97 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 752 op/s 2024-04-12T21:46:50.700 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:50 smithi177 ceph-mon[26903]: pgmap v280: 97 pgs: 97 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 752 op/s 2024-04-12T21:46:51.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 61967 19.25 MB/sec warmup 91 sec latency 21.683 ms 2024-04-12T21:46:52.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 62805 19.26 MB/sec warmup 92 sec latency 20.630 ms 2024-04-12T21:46:52.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:52 smithi123 ceph-mon[24842]: pgmap v281: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 909 op/s 2024-04-12T21:46:53.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:52 smithi177 ceph-mon[26903]: pgmap v281: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 909 op/s 2024-04-12T21:46:53.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 63511 19.35 MB/sec warmup 93 sec latency 29.403 ms 2024-04-12T21:46:54.457 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:46:54.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 63974 19.31 MB/sec warmup 94 sec latency 33.612 ms 2024-04-12T21:46:54.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:54 smithi123 ceph-mon[24842]: pgmap v282: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 936 op/s 2024-04-12T21:46:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:54 smithi177 ceph-mon[26903]: pgmap v282: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 936 op/s 2024-04-12T21:46:55.361 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (6m) 97s ago 7m 24.3M - ba2b418f427c 82f46b8aa823 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (7m) 97s ago 7m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (7m) 99s ago 7m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (6m) 97s ago 7m 52.5M - 8.3.5 dad864ee21e9 6db28cef8fcc 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (2m) 97s ago 2m 14.9M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (2m) 99s ago 2m 12.2M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:9283 running (9m) 97s ago 9m 460M - 17.2.0 e1d6a67b021e 6709173bac3d 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (7m) 99s ago 7m 412M - 17.2.0 e1d6a67b021e 632e1f1077ac 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (9m) 97s ago 9m 47.6M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (7m) 99s ago 7m 37.2M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (108s) 97s ago 107s 46.8M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (102s) 99s ago 102s 46.6M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:46:55.362 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 97s ago 7m 20.0M - 1dbe0e931976 934332c711eb 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (7m) 99s ago 7m 17.9M - 1dbe0e931976 86769ff9a983 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (6m) 97s ago 6m 42.8M 3216M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (6m) 97s ago 6m 43.0M 3216M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (5m) 97s ago 5m 44.2M 3216M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (5m) 97s ago 5m 40.0M 3216M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (4m) 99s ago 4m 42.3M 3984M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (4m) 99s ago 4m 38.6M 3984M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (4m) 99s ago 4m 42.0M 3984M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (3m) 99s ago 3m 40.5M 3984M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:46:55.363 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (6m) 97s ago 6m 62.8M - 514e6a882f6e 9086f9b4ca30 2024-04-12T21:46:55.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 64403 19.22 MB/sec warmup 95 sec latency 33.672 ms 2024-04-12T21:46:55.833 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:55 smithi123 ceph-mon[24842]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:46:55.945 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:46:55.946 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:46:55.946 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:46:55.946 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:46:55.946 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:46:55.946 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:46:55.946 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-12T21:46:55.946 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:46:55.946 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:46:56.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:55 smithi177 ceph-mon[26903]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:56.405 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:46:56.405 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:46:56.406 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:46:56.406 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-04-12T21:46:56.406 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "0/23 daemons upgraded", 2024-04-12T21:46:56.406 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec image on host smithi177" 2024-04-12T21:46:56.406 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:46:56.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 64808 19.10 MB/sec warmup 96 sec latency 55.936 ms 2024-04-12T21:46:56.845 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:56 smithi123 ceph-mon[24842]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:56.845 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:56 smithi123 ceph-mon[24842]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:56.845 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:56 smithi123 ceph-mon[24842]: pgmap v283: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 843 op/s 2024-04-12T21:46:56.845 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:56 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/843270354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:46:56.954 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-04-12T21:46:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:56 smithi177 ceph-mon[26903]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:56 smithi177 ceph-mon[26903]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:56 smithi177 ceph-mon[26903]: pgmap v283: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 843 op/s 2024-04-12T21:46:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:56 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/843270354' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:46:57.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 65346 19.02 MB/sec warmup 97 sec latency 30.035 ms 2024-04-12T21:46:57.873 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:57 smithi123 ceph-mon[24842]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:57.873 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:57 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3142322737' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:46:58.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:57 smithi177 ceph-mon[26903]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:46:58.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:57 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3142322737' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:46:58.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 65844 18.94 MB/sec warmup 98 sec latency 30.237 ms 2024-04-12T21:46:58.880 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:46:58 smithi123 ceph-mon[24842]: pgmap v284: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.0 MiB/s wr, 814 op/s 2024-04-12T21:46:59.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:46:58 smithi177 ceph-mon[26903]: pgmap v284: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.0 MiB/s wr, 814 op/s 2024-04-12T21:46:59.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 66430 18.88 MB/sec warmup 99 sec latency 45.807 ms 2024-04-12T21:47:00.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 66996 18.91 MB/sec warmup 100 sec latency 32.663 ms 2024-04-12T21:47:00.674 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:00 smithi123 ceph-mon[24842]: pgmap v285: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 766 op/s 2024-04-12T21:47:00.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:00 smithi177 ceph-mon[26903]: pgmap v285: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 766 op/s 2024-04-12T21:47:01.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 67509 18.92 MB/sec warmup 101 sec latency 98.401 ms 2024-04-12T21:47:02.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 68053 18.86 MB/sec warmup 102 sec latency 38.556 ms 2024-04-12T21:47:02.692 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:02 smithi123 ceph-mon[24842]: pgmap v286: 97 pgs: 97 active+clean; 323 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 9.2 MiB/s wr, 867 op/s 2024-04-12T21:47:03.029 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:02 smithi177 ceph-mon[26903]: pgmap v286: 97 pgs: 97 active+clean; 323 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 9.2 MiB/s wr, 867 op/s 2024-04-12T21:47:03.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 68608 18.78 MB/sec warmup 103 sec latency 32.917 ms 2024-04-12T21:47:04.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 69209 18.72 MB/sec warmup 104 sec latency 35.408 ms 2024-04-12T21:47:04.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:04 smithi123 ceph-mon[24842]: pgmap v287: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 1.01k op/s 2024-04-12T21:47:04.910 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:04 smithi177 ceph-mon[26903]: pgmap v287: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 1.01k op/s 2024-04-12T21:47:05.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 69855 18.67 MB/sec warmup 105 sec latency 30.779 ms 2024-04-12T21:47:06.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 70478 18.71 MB/sec warmup 106 sec latency 27.149 ms 2024-04-12T21:47:06.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:06 smithi123 ceph-mon[24842]: pgmap v288: 97 pgs: 97 active+clean; 342 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 915 op/s 2024-04-12T21:47:06.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:06 smithi123 ceph-mon[24842]: Upgrade: Updating mgr.smithi177.whybza 2024-04-12T21:47:06.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:06.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:47:06.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:47:06.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:06 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:47:06.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:06 smithi123 ceph-mon[24842]: Deploying daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:47:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:06 smithi177 ceph-mon[26903]: pgmap v288: 97 pgs: 97 active+clean; 342 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 915 op/s 2024-04-12T21:47:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:06 smithi177 ceph-mon[26903]: Upgrade: Updating mgr.smithi177.whybza 2024-04-12T21:47:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:47:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:47:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:06 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:47:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:06 smithi177 ceph-mon[26903]: Deploying daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:47:07.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 71185 18.79 MB/sec warmup 107 sec latency 19.526 ms 2024-04-12T21:47:08.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 71814 18.75 MB/sec warmup 108 sec latency 21.900 ms 2024-04-12T21:47:08.777 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:08 smithi123 ceph-mon[24842]: pgmap v289: 97 pgs: 97 active+clean; 342 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 7.6 MiB/s wr, 877 op/s 2024-04-12T21:47:08.887 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:08 smithi177 ceph-mon[26903]: pgmap v289: 97 pgs: 97 active+clean; 342 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 7.6 MiB/s wr, 877 op/s 2024-04-12T21:47:09.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 72624 18.71 MB/sec warmup 109 sec latency 24.955 ms 2024-04-12T21:47:10.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 73435 18.69 MB/sec warmup 110 sec latency 37.239 ms 2024-04-12T21:47:10.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:10 smithi123 ceph-mon[24842]: pgmap v290: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.2 MiB/s wr, 753 op/s 2024-04-12T21:47:10.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:10.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:47:10.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:47:10.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:10 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:47:10.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:10 smithi177 ceph-mon[26903]: pgmap v290: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.2 MiB/s wr, 753 op/s 2024-04-12T21:47:10.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:10.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:47:10.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:47:10.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:10 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:47:11.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 73943 18.71 MB/sec warmup 111 sec latency 56.346 ms 2024-04-12T21:47:12.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 74427 18.71 MB/sec warmup 112 sec latency 63.021 ms 2024-04-12T21:47:12.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:12 smithi123 ceph-mon[24842]: pgmap v291: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 958 op/s 2024-04-12T21:47:12.925 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:12 smithi177 ceph-mon[26903]: pgmap v291: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 958 op/s 2024-04-12T21:47:13.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 74863 18.67 MB/sec warmup 113 sec latency 37.262 ms 2024-04-12T21:47:14.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 75144 18.55 MB/sec warmup 114 sec latency 68.337 ms 2024-04-12T21:47:14.883 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:14 smithi123 ceph-mon[24842]: pgmap v292: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 9.5 MiB/s wr, 991 op/s 2024-04-12T21:47:15.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:14 smithi177 ceph-mon[26903]: pgmap v292: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 9.5 MiB/s wr, 991 op/s 2024-04-12T21:47:15.490 INFO:teuthology.orchestra.run.smithi123.stdout: 5 75453 18.44 MB/sec warmup 115 sec latency 100.929 ms 2024-04-12T21:47:16.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 76170 18.40 MB/sec warmup 116 sec latency 64.750 ms 2024-04-12T21:47:16.651 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:16 smithi123 ceph-mon[24842]: pgmap v293: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.7 MiB/s wr, 789 op/s 2024-04-12T21:47:16.932 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:16 smithi177 ceph-mon[26903]: pgmap v293: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.7 MiB/s wr, 789 op/s 2024-04-12T21:47:17.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 76937 18.39 MB/sec warmup 117 sec latency 22.409 ms 2024-04-12T21:47:18.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 77499 18.41 MB/sec warmup 118 sec latency 41.851 ms 2024-04-12T21:47:18.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:18 smithi123 ceph-mon[24842]: pgmap v294: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 730 op/s 2024-04-12T21:47:18.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:18 smithi177 ceph-mon[26903]: pgmap v294: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 730 op/s 2024-04-12T21:47:19.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 78053 18.45 MB/sec warmup 119 sec latency 36.058 ms 2024-04-12T21:47:19.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:19 smithi123 ceph-mon[24842]: Standby manager daemon smithi177.whybza restarted 2024-04-12T21:47:19.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:19 smithi123 ceph-mon[24842]: Standby manager daemon smithi177.whybza started 2024-04-12T21:47:19.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:19 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:47:19.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:19 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:47:19.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:19 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:47:19.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:19 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:47:19.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:19 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza restarted 2024-04-12T21:47:19.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:19 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza started 2024-04-12T21:47:19.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:19 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:47:19.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:19 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:47:19.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:19 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:47:19.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:19 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:47:20.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:20 smithi177 ceph-mon[26903]: pgmap v295: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 748 op/s 2024-04-12T21:47:20.701 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:20 smithi177 ceph-mon[26903]: mgrmap e22: smithi123.psjrfs(active, since 7m), standbys: smithi177.whybza 2024-04-12T21:47:20.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:20 smithi123 ceph-mon[24842]: pgmap v295: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 748 op/s 2024-04-12T21:47:20.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:20 smithi123 ceph-mon[24842]: mgrmap e22: smithi123.psjrfs(active, since 7m), standbys: smithi177.whybza 2024-04-12T21:47:21.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 78905 6.01 MB/sec execute 1 sec latency 76.935 ms 2024-04-12T21:47:22.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 79439 8.47 MB/sec execute 2 sec latency 39.316 ms 2024-04-12T21:47:22.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:22 smithi123 ceph-mon[24842]: pgmap v296: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 858 op/s 2024-04-12T21:47:22.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:22 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:22.915 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:22 smithi177 ceph-mon[26903]: pgmap v296: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.1 MiB/s wr, 858 op/s 2024-04-12T21:47:22.915 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:22 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:23.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 80096 10.54 MB/sec execute 3 sec latency 33.061 ms 2024-04-12T21:47:24.335 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:24.335 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:47:24.335 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:24 smithi123 ceph-mon[24842]: pgmap v297: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.1 MiB/s wr, 747 op/s 2024-04-12T21:47:24.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:24 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:47:24.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 80773 12.18 MB/sec execute 4 sec latency 49.371 ms 2024-04-12T21:47:24.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:24.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:47:24.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:24 smithi177 ceph-mon[26903]: pgmap v297: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.1 MiB/s wr, 747 op/s 2024-04-12T21:47:24.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:24 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:47:25.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 81476 15.76 MB/sec execute 5 sec latency 18.936 ms 2024-04-12T21:47:26.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 81969 15.83 MB/sec execute 6 sec latency 31.526 ms 2024-04-12T21:47:26.864 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:26 smithi123 ceph-mon[24842]: pgmap v298: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.7 MiB/s wr, 694 op/s 2024-04-12T21:47:27.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:26 smithi177 ceph-mon[26903]: pgmap v298: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.7 MiB/s wr, 694 op/s 2024-04-12T21:47:27.346 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:47:27.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 82290 14.12 MB/sec execute 7 sec latency 64.409 ms 2024-04-12T21:47:28.262 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:47:28.262 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (7m) 2m ago 8m 24.3M - ba2b418f427c 82f46b8aa823 2024-04-12T21:47:28.262 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (8m) 2m ago 8m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:47:28.262 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (7m) 5s ago 7m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:47:28.262 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (6m) 2m ago 8m 52.5M - 8.3.5 dad864ee21e9 6db28cef8fcc 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (2m) 2m ago 2m 14.9M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (2m) 5s ago 2m 12.6M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e 6709173bac3d 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (18s) 5s ago 7m 50.4M - 19.0.0-2053-gb277feb2 668895944f0e 344eee5ab888 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (9m) 2m ago 9m 47.6M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (7m) 5s ago 7m 39.8M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (2m) 2m ago 2m 46.8M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (2m) 5s ago 2m 75.8M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (8m) 2m ago 8m 20.0M - 1dbe0e931976 934332c711eb 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (7m) 5s ago 7m 19.1M - 1dbe0e931976 86769ff9a983 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (7m) 2m ago 7m 42.8M 3216M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (6m) 2m ago 6m 43.0M 3216M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (6m) 2m ago 6m 44.2M 3216M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:47:28.263 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (5m) 2m ago 5m 40.0M 3216M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:47:28.264 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (5m) 5s ago 5m 262M 3984M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:47:28.264 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (4m) 5s ago 4m 299M 3984M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:47:28.264 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (4m) 5s ago 4m 271M 3984M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:47:28.264 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (4m) 5s ago 4m 377M 3984M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:47:28.264 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (7m) 2m ago 7m 62.8M - 514e6a882f6e 9086f9b4ca30 2024-04-12T21:47:28.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 82889 13.94 MB/sec execute 8 sec latency 37.607 ms 2024-04-12T21:47:28.638 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:47:28.638 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: pgmap v299: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.2 MiB/s wr, 691 op/s 2024-04-12T21:47:28.710 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:47:28.710 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:28.710 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:28.710 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:28.710 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:47:28.710 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr fail", "who": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:47:28.710 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:28 smithi123 ceph-mon[24842]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T21:47:28.847 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:47:28.847 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:47:28.847 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-12T21:47:28.848 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:47:28.849 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:47:28.849 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: pgmap v299: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.2 MiB/s wr, 691 op/s 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr fail", "who": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:47:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:28 smithi177 ceph-mon[26903]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T21:47:29.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 83549 14.07 MB/sec execute 9 sec latency 27.142 ms 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: Upgrade: Need to upgrade myself (mgr.smithi123.psjrfs) 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: Failing over to other MGR 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/1926231335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr metadata", "who": "smithi177.whybza", "id": "smithi177.whybza"}]: dispatch 2024-04-12T21:47:29.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "mgr fail", "who": "smithi123.psjrfs"}]': finished 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: mgrmap e23: smithi177.whybza(active, starting, since 0.997243s) 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:47:29.644 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: Manager daemon smithi177.whybza is now available 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi177.whybza/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi177.whybza/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi177.whybza/trash_purge_schedule"}]: dispatch 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi177.whybza/trash_purge_schedule"}]: dispatch 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:29.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:29 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:30.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: Upgrade: Need to upgrade myself (mgr.smithi123.psjrfs) 2024-04-12T21:47:30.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: Failing over to other MGR 2024-04-12T21:47:30.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/1926231335' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr metadata", "who": "smithi177.whybza", "id": "smithi177.whybza"}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:47:30.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.14204 172.21.15.123:0/1703730878' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "mgr fail", "who": "smithi123.psjrfs"}]': finished 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: mgrmap e23: smithi177.whybza(active, starting, since 0.997243s) 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: Manager daemon smithi177.whybza is now available 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi177.whybza/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi177.whybza/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi177.whybza/trash_purge_schedule"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi177.whybza/trash_purge_schedule"}]: dispatch 2024-04-12T21:47:30.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:30.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:29 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:30.077 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:47:30.077 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:47:30.077 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:47:30.078 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:47:30.078 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-04-12T21:47:30.078 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "1/23 daemons upgraded", 2024-04-12T21:47:30.078 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-04-12T21:47:30.078 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:47:30.078 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:47:30.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 84069 13.75 MB/sec execute 10 sec latency 35.463 ms 2024-04-12T21:47:30.774 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-04-12T21:47:31.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:31 smithi123 ceph-mon[24842]: Deploying cephadm binary to smithi177 2024-04-12T21:47:31.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:31 smithi123 ceph-mon[24842]: Deploying cephadm binary to smithi123 2024-04-12T21:47:31.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:31 smithi123 ceph-mon[24842]: mgrmap e24: smithi177.whybza(active, since 1.99934s) 2024-04-12T21:47:31.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:31 smithi123 ceph-mon[24842]: from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:47:31.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:31 smithi123 ceph-mon[24842]: pgmap v3: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-12T21:47:31.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:31 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/886050083' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:47:31.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 84480 13.78 MB/sec execute 11 sec latency 54.258 ms 2024-04-12T21:47:31.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:31 smithi177 ceph-mon[26903]: Deploying cephadm binary to smithi177 2024-04-12T21:47:31.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:31 smithi177 ceph-mon[26903]: Deploying cephadm binary to smithi123 2024-04-12T21:47:31.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:31 smithi177 ceph-mon[26903]: mgrmap e24: smithi177.whybza(active, since 1.99934s) 2024-04-12T21:47:31.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:31 smithi177 ceph-mon[26903]: from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:47:31.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:31 smithi177 ceph-mon[26903]: pgmap v3: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-12T21:47:31.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:31 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/886050083' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:47:32.191 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:32 smithi123 ceph-mon[24842]: [12/Apr/2024:21:47:30] ENGINE Bus STARTING 2024-04-12T21:47:32.191 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:32 smithi123 ceph-mon[24842]: [12/Apr/2024:21:47:30] ENGINE Serving on http://172.21.15.177:8765 2024-04-12T21:47:32.191 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:32 smithi123 ceph-mon[24842]: [12/Apr/2024:21:47:30] ENGINE Serving on https://172.21.15.177:7150 2024-04-12T21:47:32.191 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:32 smithi123 ceph-mon[24842]: [12/Apr/2024:21:47:30] ENGINE Bus STARTED 2024-04-12T21:47:32.191 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:32 smithi123 ceph-mon[24842]: pgmap v4: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-12T21:47:32.192 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:32 smithi123 ceph-mon[24842]: mgrmap e25: smithi177.whybza(active, since 3s) 2024-04-12T21:47:32.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 85001 14.50 MB/sec execute 12 sec latency 31.992 ms 2024-04-12T21:47:32.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:32 smithi177 ceph-mon[26903]: [12/Apr/2024:21:47:30] ENGINE Bus STARTING 2024-04-12T21:47:32.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:32 smithi177 ceph-mon[26903]: [12/Apr/2024:21:47:30] ENGINE Serving on http://172.21.15.177:8765 2024-04-12T21:47:32.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:32 smithi177 ceph-mon[26903]: [12/Apr/2024:21:47:30] ENGINE Serving on https://172.21.15.177:7150 2024-04-12T21:47:32.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:32 smithi177 ceph-mon[26903]: [12/Apr/2024:21:47:30] ENGINE Bus STARTED 2024-04-12T21:47:32.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:32 smithi177 ceph-mon[26903]: pgmap v4: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-12T21:47:32.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:32 smithi177 ceph-mon[26903]: mgrmap e25: smithi177.whybza(active, since 3s) 2024-04-12T21:47:33.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 85527 14.68 MB/sec execute 13 sec latency 26.663 ms 2024-04-12T21:47:34.221 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:34 smithi123 ceph-mon[24842]: pgmap v5: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-12T21:47:34.221 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:34 smithi123 ceph-mon[24842]: mgrmap e26: smithi177.whybza(active, since 5s) 2024-04-12T21:47:34.491 INFO:teuthology.orchestra.run.smithi123.stdout: 5 85967 14.07 MB/sec execute 14 sec latency 34.396 ms 2024-04-12T21:47:34.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:34 smithi177 ceph-mon[26903]: pgmap v5: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-12T21:47:34.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:34 smithi177 ceph-mon[26903]: mgrmap e26: smithi177.whybza(active, since 5s) 2024-04-12T21:47:35.486 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:35 smithi123 ceph-mon[24842]: pgmap v6: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-12T21:47:35.494 INFO:teuthology.orchestra.run.smithi123.stdout: 5 86418 13.78 MB/sec execute 15 sec latency 25.136 ms 2024-04-12T21:47:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:35 smithi177 ceph-mon[26903]: pgmap v6: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-04-12T21:47:36.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 86891 13.54 MB/sec execute 16 sec latency 72.724 ms 2024-04-12T21:47:37.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 87441 13.31 MB/sec execute 17 sec latency 59.940 ms 2024-04-12T21:47:38.281 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:38 smithi123 ceph-mon[24842]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 5.8 MiB/s wr, 515 op/s 2024-04-12T21:47:38.425 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:38 smithi177 ceph-mon[26903]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 5.8 MiB/s wr, 515 op/s 2024-04-12T21:47:38.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 87917 13.39 MB/sec execute 18 sec latency 29.867 ms 2024-04-12T21:47:39.287 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:39 smithi123 ceph-mon[24842]: Standby manager daemon smithi123.psjrfs started 2024-04-12T21:47:39.287 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:39 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.123:0/2117837865' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.psjrfs/crt"}]: dispatch 2024-04-12T21:47:39.287 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:39 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.123:0/2117837865' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:47:39.287 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:39 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.123:0/2117837865' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.psjrfs/key"}]: dispatch 2024-04-12T21:47:39.287 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:39 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.123:0/2117837865' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:47:39.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:39 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:39.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:39 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:39.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 88458 13.99 MB/sec execute 19 sec latency 45.272 ms 2024-04-12T21:47:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:39 smithi177 ceph-mon[26903]: Standby manager daemon smithi123.psjrfs started 2024-04-12T21:47:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:39 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.123:0/2117837865' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.psjrfs/crt"}]: dispatch 2024-04-12T21:47:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:39 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.123:0/2117837865' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:47:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:39 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.123:0/2117837865' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.psjrfs/key"}]: dispatch 2024-04-12T21:47:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:39 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.123:0/2117837865' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:47:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:39 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:39 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:40.296 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:40 smithi123 ceph-mon[24842]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 4.5 MiB/s wr, 400 op/s 2024-04-12T21:47:40.296 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:40 smithi123 ceph-mon[24842]: mgrmap e27: smithi177.whybza(active, since 11s), standbys: smithi123.psjrfs 2024-04-12T21:47:40.296 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:40 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:47:40.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 88951 14.25 MB/sec execute 20 sec latency 32.800 ms 2024-04-12T21:47:40.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:40 smithi177 ceph-mon[26903]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 4.5 MiB/s wr, 400 op/s 2024-04-12T21:47:40.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:40 smithi177 ceph-mon[26903]: mgrmap e27: smithi177.whybza(active, since 11s), standbys: smithi123.psjrfs 2024-04-12T21:47:40.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:40 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:47:41.469 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:41 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:41.469 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:41 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:41.470 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:41 smithi123 ceph-mon[24842]: pgmap v9: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 7.1 MiB/s wr, 644 op/s 2024-04-12T21:47:41.475 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:41 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:41.475 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:41 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:41.475 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:41 smithi177 ceph-mon[26903]: pgmap v9: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 7.1 MiB/s wr, 644 op/s 2024-04-12T21:47:41.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 89373 13.81 MB/sec execute 21 sec latency 37.744 ms 2024-04-12T21:47:42.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 89813 13.66 MB/sec execute 22 sec latency 32.601 ms 2024-04-12T21:47:43.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 90269 13.41 MB/sec execute 23 sec latency 45.887 ms 2024-04-12T21:47:43.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:43 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:43.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:43 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:43.989 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:43 smithi123 ceph-mon[24842]: pgmap v10: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.5 MiB/s wr, 590 op/s 2024-04-12T21:47:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:43 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:43 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:43 smithi177 ceph-mon[26903]: pgmap v10: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.5 MiB/s wr, 590 op/s 2024-04-12T21:47:44.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 90744 13.27 MB/sec execute 24 sec latency 44.266 ms 2024-04-12T21:47:45.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi177 to 2704M 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:47:45.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:47:45.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:47:45.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 91262 13.21 MB/sec execute 25 sec latency 48.085 ms 2024-04-12T21:47:45.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi177 to 2704M 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:47:45.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: Adjusting osd_memory_target on smithi123 to 1936M 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: Updating smithi177:/etc/ceph/ceph.conf 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: pgmap v11: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.6 MiB/s wr, 599 op/s 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:46 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 91628 13.25 MB/sec execute 26 sec latency 31.071 ms 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: Adjusting osd_memory_target on smithi123 to 1936M 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: Updating smithi177:/etc/ceph/ceph.conf 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: pgmap v11: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.6 MiB/s wr, 599 op/s 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:46 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:47:47.302 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:47 smithi123 ceph-mon[24842]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:47:47.302 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:47 smithi123 ceph-mon[24842]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:47:47.302 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:47 smithi123 ceph-mon[24842]: Reconfiguring alertmanager.smithi123 (dependencies changed)... 2024-04-12T21:47:47.302 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:47 smithi123 ceph-mon[24842]: Reconfiguring daemon alertmanager.smithi123 on smithi123 2024-04-12T21:47:47.492 INFO:teuthology.orchestra.run.smithi123.stdout: 5 92327 13.80 MB/sec execute 27 sec latency 30.263 ms 2024-04-12T21:47:47.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:47 smithi177 ceph-mon[26903]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:47:47.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:47 smithi177 ceph-mon[26903]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:47:47.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:47 smithi177 ceph-mon[26903]: Reconfiguring alertmanager.smithi123 (dependencies changed)... 2024-04-12T21:47:47.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:47 smithi177 ceph-mon[26903]: Reconfiguring daemon alertmanager.smithi123 on smithi123 2024-04-12T21:47:48.307 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:48 smithi123 ceph-mon[24842]: pgmap v12: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 8.8 MiB/s wr, 874 op/s 2024-04-12T21:47:48.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 92803 13.72 MB/sec execute 28 sec latency 26.879 ms 2024-04-12T21:47:48.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:48 smithi177 ceph-mon[26903]: pgmap v12: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 8.8 MiB/s wr, 874 op/s 2024-04-12T21:47:49.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 93306 13.55 MB/sec execute 29 sec latency 21.882 ms 2024-04-12T21:47:50.322 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:50 smithi123 ceph-mon[24842]: pgmap v13: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.5 MiB/s wr, 574 op/s 2024-04-12T21:47:50.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 93835 13.47 MB/sec execute 30 sec latency 46.378 ms 2024-04-12T21:47:50.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:50 smithi177 ceph-mon[26903]: pgmap v13: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.5 MiB/s wr, 574 op/s 2024-04-12T21:47:51.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 94812 13.76 MB/sec execute 31 sec latency 22.471 ms 2024-04-12T21:47:52.279 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:52 smithi177 ceph-mon[26903]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 880 op/s 2024-04-12T21:47:52.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:52 smithi123 ceph-mon[24842]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 880 op/s 2024-04-12T21:47:52.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 95558 14.31 MB/sec execute 32 sec latency 56.416 ms 2024-04-12T21:47:53.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 96101 14.43 MB/sec execute 33 sec latency 23.282 ms 2024-04-12T21:47:54.096 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:54 smithi123 ceph-mon[24842]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.8 MiB/s wr, 590 op/s 2024-04-12T21:47:54.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 96467 14.15 MB/sec execute 34 sec latency 31.764 ms 2024-04-12T21:47:54.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:54 smithi177 ceph-mon[26903]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.8 MiB/s wr, 590 op/s 2024-04-12T21:47:55.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 96844 13.95 MB/sec execute 35 sec latency 35.007 ms 2024-04-12T21:47:55.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:55 smithi123 ceph-mon[24842]: pgmap v16: 97 pgs: 97 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.3 MiB/s wr, 609 op/s 2024-04-12T21:47:55.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:55 smithi177 ceph-mon[26903]: pgmap v16: 97 pgs: 97 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.3 MiB/s wr, 609 op/s 2024-04-12T21:47:56.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 97300 13.87 MB/sec execute 36 sec latency 64.266 ms 2024-04-12T21:47:57.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 97820 13.77 MB/sec execute 37 sec latency 72.958 ms 2024-04-12T21:47:58.176 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:47:58 smithi123 ceph-mon[24842]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 939 op/s 2024-04-12T21:47:58.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 98352 13.73 MB/sec execute 38 sec latency 58.143 ms 2024-04-12T21:47:58.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:47:58 smithi177 ceph-mon[26903]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 939 op/s 2024-04-12T21:47:59.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 98911 14.00 MB/sec execute 39 sec latency 34.475 ms 2024-04-12T21:48:00.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:00 smithi123 ceph-mon[24842]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.4 MiB/s wr, 664 op/s 2024-04-12T21:48:00.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:00 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:48:00.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 99460 14.11 MB/sec execute 40 sec latency 22.255 ms 2024-04-12T21:48:00.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:00 smithi177 ceph-mon[26903]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.4 MiB/s wr, 664 op/s 2024-04-12T21:48:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:00 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:48:01.336 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:48:01.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 99805 14.02 MB/sec execute 41 sec latency 39.924 ms 2024-04-12T21:48:02.213 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:02 smithi123 ceph-mon[24842]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 962 op/s 2024-04-12T21:48:02.386 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:48:02.386 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (7m) 19s ago 9m 28.2M - 0.23.0 ba2b418f427c 82f46b8aa823 2024-04-12T21:48:02.386 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (9m) 19s ago 9m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:48:02.386 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (8m) 23s ago 8m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:48:02.386 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (7m) 19s ago 8m 52.5M - 8.3.5 dad864ee21e9 6db28cef8fcc 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (3m) 19s ago 3m 36.3M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (3m) 23s ago 3m 13.0M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:9283 running (10m) 19s ago 10m 307M - 17.2.0 e1d6a67b021e 6709173bac3d 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (52s) 23s ago 8m 489M - 19.0.0-2053-gb277feb2 668895944f0e 344eee5ab888 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (10m) 19s ago 10m 49.7M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (8m) 23s ago 8m 41.3M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (2m) 19s ago 2m 244M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (2m) 23s ago 2m 78.2M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (8m) 19s ago 8m 19.9M - 1.3.1 1dbe0e931976 934332c711eb 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (8m) 23s ago 8m 17.7M - 1.3.1 1dbe0e931976 86769ff9a983 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (7m) 19s ago 7m 345M 1936M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (7m) 19s ago 7m 261M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:48:02.387 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (6m) 19s ago 6m 205M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:48:02.388 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (6m) 19s ago 6m 295M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:48:02.388 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (5m) 23s ago 5m 319M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:48:02.388 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (5m) 23s ago 5m 339M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:48:02.388 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (5m) 23s ago 5m 275M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:48:02.388 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (4m) 23s ago 4m 432M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:48:02.388 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (8m) 19s ago 8m 67.1M - 2.33.4 514e6a882f6e 9086f9b4ca30 2024-04-12T21:48:02.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 100083 13.78 MB/sec execute 42 sec latency 33.368 ms 2024-04-12T21:48:02.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:02 smithi177 ceph-mon[26903]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 962 op/s 2024-04-12T21:48:03.032 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:48:03.032 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:48:03.032 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:48:03.032 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:48:03.032 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:48:03.032 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:48:03.033 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:48:03.222 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:03 smithi123 ceph-mon[24842]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:03.222 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:03 smithi123 ceph-mon[24842]: from='client.24475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:03.223 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:03 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/865652686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:48:03.223 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:03 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:03.223 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:03 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:03.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 100407 13.64 MB/sec execute 43 sec latency 52.182 ms 2024-04-12T21:48:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:03 smithi177 ceph-mon[26903]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:03 smithi177 ceph-mon[26903]: from='client.24475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:03 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/865652686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:48:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:03 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:03 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:03.590 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:48:03.590 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:48:03.591 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:48:03.591 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:48:03.591 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-04-12T21:48:03.591 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "1/23 daemons upgraded", 2024-04-12T21:48:03.591 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-04-12T21:48:03.591 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:48:03.591 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:48:04.274 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:04 smithi123 ceph-mon[24842]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:04.274 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:04 smithi123 ceph-mon[24842]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 656 op/s 2024-04-12T21:48:04.275 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:04 smithi123 ceph-mon[24842]: Reconfiguring grafana.smithi123 (dependencies changed)... 2024-04-12T21:48:04.275 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:04 smithi123 ceph-mon[24842]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-12T21:48:04.275 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:04 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:04.275 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:04 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:04.275 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:04 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T21:48:04.290 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-04-12T21:48:04.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 100875 13.53 MB/sec execute 44 sec latency 60.567 ms 2024-04-12T21:48:04.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:04 smithi177 ceph-mon[26903]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:04.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:04 smithi177 ceph-mon[26903]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 656 op/s 2024-04-12T21:48:04.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:04 smithi177 ceph-mon[26903]: Reconfiguring grafana.smithi123 (dependencies changed)... 2024-04-12T21:48:04.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:04 smithi177 ceph-mon[26903]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-12T21:48:04.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:04 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:04.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:04 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:04.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:04 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T21:48:05.281 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:05 smithi123 ceph-mon[24842]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T21:48:05.281 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:05 smithi123 ceph-mon[24842]: Reconfiguring daemon grafana.smithi123 on smithi123 2024-04-12T21:48:05.281 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:05 smithi123 ceph-mon[24842]: from='client.24485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:05.281 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:05 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/26567145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:48:05.499 INFO:teuthology.orchestra.run.smithi123.stdout: 5 101509 13.50 MB/sec execute 45 sec latency 27.160 ms 2024-04-12T21:48:05.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:05 smithi177 ceph-mon[26903]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T21:48:05.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:05 smithi177 ceph-mon[26903]: Reconfiguring daemon grafana.smithi123 on smithi123 2024-04-12T21:48:05.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:05 smithi177 ceph-mon[26903]: from='client.24485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:05.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:05 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/26567145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:48:06.287 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:06 smithi123 ceph-mon[24842]: pgmap v21: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 665 op/s 2024-04-12T21:48:06.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 102069 13.54 MB/sec execute 46 sec latency 39.354 ms 2024-04-12T21:48:06.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:06 smithi177 ceph-mon[26903]: pgmap v21: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 665 op/s 2024-04-12T21:48:07.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 102502 13.67 MB/sec execute 47 sec latency 35.979 ms 2024-04-12T21:48:08.360 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:08 smithi123 ceph-mon[24842]: pgmap v22: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 890 op/s 2024-04-12T21:48:08.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 102820 13.67 MB/sec execute 48 sec latency 42.586 ms 2024-04-12T21:48:08.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:08 smithi177 ceph-mon[26903]: pgmap v22: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 890 op/s 2024-04-12T21:48:09.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 103150 13.62 MB/sec execute 49 sec latency 35.484 ms 2024-04-12T21:48:10.143 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:10 smithi123 ceph-mon[24842]: pgmap v23: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.3 MiB/s wr, 551 op/s 2024-04-12T21:48:10.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 103537 13.45 MB/sec execute 50 sec latency 30.255 ms 2024-04-12T21:48:10.529 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:10 smithi177 ceph-mon[26903]: pgmap v23: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.3 MiB/s wr, 551 op/s 2024-04-12T21:48:11.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 103898 13.27 MB/sec execute 51 sec latency 34.477 ms 2024-04-12T21:48:12.162 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:12 smithi123 ceph-mon[24842]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 803 op/s 2024-04-12T21:48:12.432 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:12 smithi177 ceph-mon[26903]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 803 op/s 2024-04-12T21:48:12.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 104334 13.26 MB/sec execute 52 sec latency 52.691 ms 2024-04-12T21:48:13.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 104921 13.22 MB/sec execute 53 sec latency 59.726 ms 2024-04-12T21:48:14.184 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:14 smithi123 ceph-mon[24842]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 505 op/s 2024-04-12T21:48:14.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 105480 13.23 MB/sec execute 54 sec latency 30.169 ms 2024-04-12T21:48:14.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:14 smithi177 ceph-mon[26903]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 505 op/s 2024-04-12T21:48:15.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:15 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:48:15.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 105962 13.30 MB/sec execute 55 sec latency 31.887 ms 2024-04-12T21:48:15.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:15 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:48:16.203 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:16 smithi123 ceph-mon[24842]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 520 op/s 2024-04-12T21:48:16.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 106476 13.47 MB/sec execute 56 sec latency 30.909 ms 2024-04-12T21:48:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:16 smithi177 ceph-mon[26903]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 520 op/s 2024-04-12T21:48:17.464 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:17 smithi123 ceph-mon[24842]: pgmap v27: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 781 op/s 2024-04-12T21:48:17.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 106857 13.43 MB/sec execute 57 sec latency 31.236 ms 2024-04-12T21:48:17.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:17 smithi177 ceph-mon[26903]: pgmap v27: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 781 op/s 2024-04-12T21:48:18.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 107172 13.25 MB/sec execute 58 sec latency 44.837 ms 2024-04-12T21:48:19.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 107484 13.10 MB/sec execute 59 sec latency 124.969 ms 2024-04-12T21:48:20.166 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:20 smithi123 ceph-mon[24842]: pgmap v28: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 537 op/s 2024-04-12T21:48:20.500 INFO:teuthology.orchestra.run.smithi123.stdout: 5 107835 13.04 MB/sec execute 60 sec latency 88.750 ms 2024-04-12T21:48:20.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:20 smithi177 ceph-mon[26903]: pgmap v28: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 537 op/s 2024-04-12T21:48:21.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 108320 13.03 MB/sec execute 61 sec latency 52.421 ms 2024-04-12T21:48:22.187 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:22 smithi123 ceph-mon[24842]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 785 op/s 2024-04-12T21:48:22.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 108721 12.90 MB/sec execute 62 sec latency 53.931 ms 2024-04-12T21:48:22.504 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:22 smithi177 ceph-mon[26903]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 785 op/s 2024-04-12T21:48:23.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 109331 13.03 MB/sec execute 63 sec latency 30.916 ms 2024-04-12T21:48:23.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:23 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:23.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:23 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:23.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:23 smithi123 ceph-mon[24842]: Reconfiguring node-exporter.smithi123 (dependencies changed)... 2024-04-12T21:48:23.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:23 smithi123 ceph-mon[24842]: Reconfiguring daemon node-exporter.smithi123 on smithi123 2024-04-12T21:48:23.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:23 smithi123 ceph-mon[24842]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 533 op/s 2024-04-12T21:48:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:23 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:23 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:23 smithi177 ceph-mon[26903]: Reconfiguring node-exporter.smithi123 (dependencies changed)... 2024-04-12T21:48:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:23 smithi177 ceph-mon[26903]: Reconfiguring daemon node-exporter.smithi123 on smithi123 2024-04-12T21:48:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:23 smithi177 ceph-mon[26903]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 533 op/s 2024-04-12T21:48:24.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 109973 13.27 MB/sec execute 64 sec latency 25.366 ms 2024-04-12T21:48:25.471 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:25 smithi123 ceph-mon[24842]: pgmap v31: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 557 op/s 2024-04-12T21:48:25.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 110434 13.25 MB/sec execute 65 sec latency 53.026 ms 2024-04-12T21:48:25.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:25 smithi177 ceph-mon[26903]: pgmap v31: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 557 op/s 2024-04-12T21:48:26.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 110816 13.12 MB/sec execute 66 sec latency 22.976 ms 2024-04-12T21:48:27.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 111299 13.09 MB/sec execute 67 sec latency 31.677 ms 2024-04-12T21:48:28.177 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:28 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:28.178 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:28 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:28.178 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:28 smithi123 ceph-mon[24842]: Reconfiguring prometheus.smithi123 (dependencies changed)... 2024-04-12T21:48:28.178 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:28 smithi123 ceph-mon[24842]: pgmap v32: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 807 op/s 2024-04-12T21:48:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:28 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:28 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:28 smithi177 ceph-mon[26903]: Reconfiguring prometheus.smithi123 (dependencies changed)... 2024-04-12T21:48:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:28 smithi177 ceph-mon[26903]: pgmap v32: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 807 op/s 2024-04-12T21:48:28.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 111845 13.05 MB/sec execute 68 sec latency 39.708 ms 2024-04-12T21:48:29.185 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:29 smithi123 ceph-mon[24842]: Reconfiguring daemon prometheus.smithi123 on smithi123 2024-04-12T21:48:29.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:29 smithi177 ceph-mon[26903]: Reconfiguring daemon prometheus.smithi123 on smithi123 2024-04-12T21:48:29.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 112331 13.02 MB/sec execute 69 sec latency 50.530 ms 2024-04-12T21:48:30.205 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:30 smithi123 ceph-mon[24842]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 537 op/s 2024-04-12T21:48:30.205 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:30 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:48:30.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 112719 13.01 MB/sec execute 70 sec latency 29.554 ms 2024-04-12T21:48:30.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:30 smithi177 ceph-mon[26903]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 537 op/s 2024-04-12T21:48:30.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:30 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:48:31.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 113134 13.06 MB/sec execute 71 sec latency 57.532 ms 2024-04-12T21:48:32.258 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:32 smithi123 ceph-mon[24842]: pgmap v34: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 824 op/s 2024-04-12T21:48:32.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 113518 13.11 MB/sec execute 72 sec latency 42.993 ms 2024-04-12T21:48:32.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:32 smithi177 ceph-mon[26903]: pgmap v34: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 824 op/s 2024-04-12T21:48:33.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 113875 13.09 MB/sec execute 73 sec latency 50.345 ms 2024-04-12T21:48:34.298 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:34 smithi123 ceph-mon[24842]: pgmap v35: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 576 op/s 2024-04-12T21:48:34.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 114125 12.95 MB/sec execute 74 sec latency 41.825 ms 2024-04-12T21:48:34.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:34 smithi177 ceph-mon[26903]: pgmap v35: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 576 op/s 2024-04-12T21:48:34.885 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:48:35.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 114391 12.82 MB/sec execute 75 sec latency 42.035 ms 2024-04-12T21:48:35.564 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:35 smithi123 ceph-mon[24842]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:35.565 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:35 smithi123 ceph-mon[24842]: pgmap v36: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.4 MiB/s wr, 606 op/s 2024-04-12T21:48:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:35 smithi177 ceph-mon[26903]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:35 smithi177 ceph-mon[26903]: pgmap v36: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.4 MiB/s wr, 606 op/s 2024-04-12T21:48:35.967 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:48:35.967 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 starting - - - - 2024-04-12T21:48:35.967 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (9m) 53s ago 9m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:48:35.967 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (9m) 57s ago 9m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:48:35.967 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 starting - - - - 2024-04-12T21:48:35.967 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (3m) 53s ago 3m 36.3M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:48:35.967 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (3m) 57s ago 3m 13.0M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:48:35.967 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:9283 running (10m) 53s ago 10m 307M - 17.2.0 e1d6a67b021e 6709173bac3d 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (86s) 57s ago 9m 489M - 19.0.0-2053-gb277feb2 668895944f0e 344eee5ab888 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (10m) 53s ago 11m 49.7M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (8m) 57s ago 8m 41.3M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (3m) 53s ago 3m 244M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (3m) 57s ago 3m 78.2M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 starting - - - - 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (8m) 57s ago 8m 17.7M - 1.3.1 1dbe0e931976 86769ff9a983 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (8m) 53s ago 8m 345M 1936M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (7m) 53s ago 7m 261M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (7m) 53s ago 7m 205M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (6m) 53s ago 6m 295M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (6m) 57s ago 6m 319M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (6m) 57s ago 6m 339M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (5m) 57s ago 5m 275M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:48:35.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (5m) 57s ago 5m 432M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:48:35.969 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (8m) 53s ago 8m 67.1M - 2.33.4 514e6a882f6e 9086f9b4ca30 2024-04-12T21:48:36.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 114696 12.74 MB/sec execute 76 sec latency 71.434 ms 2024-04-12T21:48:36.574 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:36 smithi123 ceph-mon[24842]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:36.574 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:36 smithi123 ceph-mon[24842]: from='client.24499 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:36.581 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:48:36.581 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:48:36.581 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:48:36.582 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-12T21:48:36.583 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:48:36.583 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:48:36.583 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:48:36.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:36 smithi177 ceph-mon[26903]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:36.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:36 smithi177 ceph-mon[26903]: from='client.24499 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "1/23 daemons upgraded", 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:48:37.119 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:48:37.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 115044 12.69 MB/sec execute 77 sec latency 51.902 ms 2024-04-12T21:48:37.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:37 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/2832151487' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:48:37.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:37 smithi123 ceph-mon[24842]: pgmap v37: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 758 op/s 2024-04-12T21:48:37.615 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:37 smithi123 ceph-mon[24842]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:37.739 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-04-12T21:48:37.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:37 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2832151487' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:48:37.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:37 smithi177 ceph-mon[26903]: pgmap v37: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 758 op/s 2024-04-12T21:48:37.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:37 smithi177 ceph-mon[26903]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:48:38.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 115554 12.66 MB/sec execute 78 sec latency 68.429 ms 2024-04-12T21:48:38.637 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:38 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/895368328' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:48:38.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:38 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/895368328' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:48:39.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 115994 12.62 MB/sec execute 79 sec latency 49.890 ms 2024-04-12T21:48:39.666 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:39 smithi123 ceph-mon[24842]: pgmap v38: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 493 op/s 2024-04-12T21:48:39.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:39 smithi177 ceph-mon[26903]: pgmap v38: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 493 op/s 2024-04-12T21:48:40.501 INFO:teuthology.orchestra.run.smithi123.stdout: 5 116445 12.66 MB/sec execute 80 sec latency 40.905 ms 2024-04-12T21:48:41.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 116946 12.78 MB/sec execute 81 sec latency 32.398 ms 2024-04-12T21:48:42.215 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:42 smithi123 ceph-mon[24842]: pgmap v39: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 732 op/s 2024-04-12T21:48:42.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 117361 12.81 MB/sec execute 82 sec latency 29.990 ms 2024-04-12T21:48:42.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:42 smithi177 ceph-mon[26903]: pgmap v39: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 732 op/s 2024-04-12T21:48:43.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 117691 12.70 MB/sec execute 83 sec latency 58.541 ms 2024-04-12T21:48:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:44 smithi123 ceph-mon[24842]: pgmap v40: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-04-12T21:48:44.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 118153 12.65 MB/sec execute 84 sec latency 48.592 ms 2024-04-12T21:48:44.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:44 smithi177 ceph-mon[26903]: pgmap v40: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-04-12T21:48:45.275 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:45 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:48:45.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 118757 12.67 MB/sec execute 85 sec latency 50.271 ms 2024-04-12T21:48:45.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:45 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:48:46.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:46 smithi123 ceph-mon[24842]: pgmap v41: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 562 op/s 2024-04-12T21:48:46.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 119321 12.63 MB/sec execute 86 sec latency 78.776 ms 2024-04-12T21:48:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:46 smithi177 ceph-mon[26903]: pgmap v41: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 562 op/s 2024-04-12T21:48:47.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 119872 12.70 MB/sec execute 87 sec latency 36.750 ms 2024-04-12T21:48:48.308 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:48 smithi123 ceph-mon[24842]: pgmap v42: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-04-12T21:48:48.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 120345 12.77 MB/sec execute 88 sec latency 58.694 ms 2024-04-12T21:48:48.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:48 smithi177 ceph-mon[26903]: pgmap v42: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-04-12T21:48:49.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 120763 12.83 MB/sec execute 89 sec latency 30.259 ms 2024-04-12T21:48:50.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:50 smithi123 ceph-mon[24842]: pgmap v43: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 511 op/s 2024-04-12T21:48:50.502 INFO:teuthology.orchestra.run.smithi123.stdout: 5 121102 12.76 MB/sec execute 90 sec latency 40.482 ms 2024-04-12T21:48:50.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:50 smithi177 ceph-mon[26903]: pgmap v43: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 511 op/s 2024-04-12T21:48:51.503 INFO:teuthology.orchestra.run.smithi123.stdout: 5 121592 12.69 MB/sec execute 91 sec latency 26.307 ms 2024-04-12T21:48:52.344 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:52 smithi123 ceph-mon[24842]: pgmap v44: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 798 op/s 2024-04-12T21:48:52.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 122082 12.70 MB/sec execute 92 sec latency 56.487 ms 2024-04-12T21:48:52.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:52 smithi177 ceph-mon[26903]: pgmap v44: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 798 op/s 2024-04-12T21:48:53.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 122740 12.68 MB/sec execute 93 sec latency 54.208 ms 2024-04-12T21:48:54.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:54 smithi123 ceph-mon[24842]: pgmap v45: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-04-12T21:48:54.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:54 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:54.197 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:54 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:54.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 123329 12.74 MB/sec execute 94 sec latency 27.115 ms 2024-04-12T21:48:54.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:54 smithi177 ceph-mon[26903]: pgmap v45: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-04-12T21:48:54.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:54 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:54.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:54 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:55.229 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:55 smithi123 ceph-mon[24842]: Reconfiguring node-exporter.smithi177 (dependencies changed)... 2024-04-12T21:48:55.229 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:55 smithi123 ceph-mon[24842]: Reconfiguring daemon node-exporter.smithi177 on smithi177 2024-04-12T21:48:55.371 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:55 smithi177 ceph-mon[26903]: Reconfiguring node-exporter.smithi177 (dependencies changed)... 2024-04-12T21:48:55.371 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:55 smithi177 ceph-mon[26903]: Reconfiguring daemon node-exporter.smithi177 on smithi177 2024-04-12T21:48:55.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 123820 12.83 MB/sec execute 95 sec latency 44.555 ms 2024-04-12T21:48:56.248 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:56 smithi177 ceph-mon[26903]: pgmap v46: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 7.3 MiB/s wr, 646 op/s 2024-04-12T21:48:56.298 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:56 smithi123 ceph-mon[24842]: pgmap v46: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 7.3 MiB/s wr, 646 op/s 2024-04-12T21:48:56.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 124212 12.84 MB/sec execute 96 sec latency 63.814 ms 2024-04-12T21:48:57.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 124539 12.81 MB/sec execute 97 sec latency 37.335 ms 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi123.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi123.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:48:57.841 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: Upgrade: Need to upgrade myself (mgr.smithi177.whybza) 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: Upgrade: Need to upgrade myself (mgr.smithi177.whybza) 2024-04-12T21:48:57.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:48:57 smithi123 ceph-mon[24842]: pgmap v47: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.9 MiB/s wr, 768 op/s 2024-04-12T21:48:58.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:58.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:58.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi123.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi123.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:48:58.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T21:48:58.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T21:48:58.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:48:58.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:48:58.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: Upgrade: Need to upgrade myself (mgr.smithi177.whybza) 2024-04-12T21:48:58.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: Upgrade: Need to upgrade myself (mgr.smithi177.whybza) 2024-04-12T21:48:58.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:48:57 smithi177 ceph-mon[26903]: pgmap v47: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.9 MiB/s wr, 768 op/s 2024-04-12T21:48:58.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 124920 12.72 MB/sec execute 98 sec latency 52.776 ms 2024-04-12T21:48:59.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 125349 12.68 MB/sec execute 99 sec latency 36.473 ms 2024-04-12T21:49:00.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:00 smithi123 ceph-mon[24842]: pgmap v48: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 612 op/s 2024-04-12T21:49:00.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:00 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:00.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:00 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:00.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:00 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.psjrfs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:49:00.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:00 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.psjrfs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:49:00.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:00 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:49:00.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:00 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:49:00.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 125825 12.68 MB/sec execute 100 sec latency 50.721 ms 2024-04-12T21:49:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:00 smithi177 ceph-mon[26903]: pgmap v48: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 612 op/s 2024-04-12T21:49:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:00 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:00 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:00 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.psjrfs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:49:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:00 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.psjrfs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:49:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:00 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:49:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:00 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:49:01.404 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:01 smithi123 ceph-mon[24842]: Upgrade: Updating mgr.smithi123.psjrfs 2024-04-12T21:49:01.404 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:01 smithi123 ceph-mon[24842]: Deploying daemon mgr.smithi123.psjrfs on smithi123 2024-04-12T21:49:01.505 INFO:teuthology.orchestra.run.smithi123.stdout: 5 126283 12.62 MB/sec execute 101 sec latency 67.138 ms 2024-04-12T21:49:01.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:01 smithi177 ceph-mon[26903]: Upgrade: Updating mgr.smithi123.psjrfs 2024-04-12T21:49:01.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:01 smithi177 ceph-mon[26903]: Deploying daemon mgr.smithi123.psjrfs on smithi123 2024-04-12T21:49:02.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:02 smithi123 ceph-mon[24842]: pgmap v49: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 862 op/s 2024-04-12T21:49:02.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:02 smithi123 ceph-mon[24842]: mgrmap e28: smithi177.whybza(active, since 93s), standbys: smithi123.psjrfs 2024-04-12T21:49:02.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 126723 12.60 MB/sec execute 102 sec latency 31.491 ms 2024-04-12T21:49:02.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:02 smithi177 ceph-mon[26903]: pgmap v49: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 862 op/s 2024-04-12T21:49:02.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:02 smithi177 ceph-mon[26903]: mgrmap e28: smithi177.whybza(active, since 93s), standbys: smithi123.psjrfs 2024-04-12T21:49:03.447 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:03 smithi123 ceph-mon[24842]: pgmap v50: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.6 MiB/s wr, 575 op/s 2024-04-12T21:49:03.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 127348 12.75 MB/sec execute 103 sec latency 33.292 ms 2024-04-12T21:49:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:03 smithi177 ceph-mon[26903]: pgmap v50: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.6 MiB/s wr, 575 op/s 2024-04-12T21:49:04.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 127784 12.78 MB/sec execute 104 sec latency 76.278 ms 2024-04-12T21:49:05.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 128134 12.76 MB/sec execute 105 sec latency 38.809 ms 2024-04-12T21:49:05.532 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:05 smithi123 ceph-mon[24842]: pgmap v51: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 630 op/s 2024-04-12T21:49:05.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:05 smithi177 ceph-mon[26903]: pgmap v51: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 630 op/s 2024-04-12T21:49:06.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 128417 12.66 MB/sec execute 106 sec latency 67.279 ms 2024-04-12T21:49:07.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 128677 12.57 MB/sec execute 107 sec latency 38.700 ms 2024-04-12T21:49:08.085 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:07 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:08.085 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:07 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:08.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:07 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:49:08.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:07 smithi123 ceph-mon[24842]: pgmap v52: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 766 op/s 2024-04-12T21:49:08.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:07 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:08.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:07 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:07 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:07 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:07 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:49:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:07 smithi177 ceph-mon[26903]: pgmap v52: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 766 op/s 2024-04-12T21:49:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:07 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:07 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:08.402 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:49:08.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 128880 12.51 MB/sec execute 108 sec latency 84.874 ms 2024-04-12T21:49:09.499 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:49:09.499 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 starting - - - - 2024-04-12T21:49:09.499 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (10m) 86s ago 10m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:49:09.499 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (9m) 90s ago 9m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:49:09.499 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 starting - - - - 2024-04-12T21:49:09.499 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (4m) 86s ago 4m 36.3M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (4m) 90s ago 4m 13.0M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 starting - - - - 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (119s) 90s ago 9m 489M - 19.0.0-2053-gb277feb2 668895944f0e 344eee5ab888 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (11m) 86s ago 11m 49.7M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (9m) 90s ago 9m 41.3M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (4m) 86s ago 4m 244M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (3m) 90s ago 3m 78.2M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 starting - - - - 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 starting - - - - 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (8m) 86s ago 8m 345M 1936M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (8m) 86s ago 8m 261M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:49:09.500 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (7m) 86s ago 7m 205M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:49:09.501 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (7m) 86s ago 7m 295M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:49:09.501 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (7m) 90s ago 7m 319M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:49:09.501 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (6m) 90s ago 6m 339M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:49:09.501 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (6m) 90s ago 6m 275M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:49:09.501 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (5m) 90s ago 5m 432M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:49:09.501 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 starting - - - - 2024-04-12T21:49:09.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 129125 12.46 MB/sec execute 109 sec latency 90.752 ms 2024-04-12T21:49:10.134 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:49:10.134 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:49:10.134 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:49:10.134 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:49:10.134 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:49:10.134 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T21:49:10.134 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:49:10.135 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:49:10.360 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:10 smithi123 ceph-mon[24842]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:10.360 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:10 smithi123 ceph-mon[24842]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:10.361 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:10 smithi123 ceph-mon[24842]: pgmap v53: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 527 op/s 2024-04-12T21:49:10.361 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:10 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/129577507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:49:10.443 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:10 smithi177 ceph-mon[26903]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:10.443 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:10 smithi177 ceph-mon[26903]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:10.443 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:10 smithi177 ceph-mon[26903]: pgmap v53: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 527 op/s 2024-04-12T21:49:10.443 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:10 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/129577507' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:49:10.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 129395 12.40 MB/sec execute 110 sec latency 72.720 ms 2024-04-12T21:49:10.675 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:49:10.676 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:49:10.676 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:49:10.676 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:49:10.676 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-04-12T21:49:10.676 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "", 2024-04-12T21:49:10.676 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mgr daemons", 2024-04-12T21:49:10.676 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:49:10.676 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:49:11.297 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-04-12T21:49:11.396 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:11 smithi123 ceph-mon[24842]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:11.396 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:11 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:11.396 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:11 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:11.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 129670 12.33 MB/sec execute 111 sec latency 80.152 ms 2024-04-12T21:49:11.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:11 smithi177 ceph-mon[26903]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:11.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:11 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:11.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:11 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:12.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:12 smithi123 ceph-mon[24842]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:12.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:12 smithi123 ceph-mon[24842]: pgmap v54: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 702 op/s 2024-04-12T21:49:12.174 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:12 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3981741676' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:49:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:12 smithi177 ceph-mon[26903]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:12 smithi177 ceph-mon[26903]: pgmap v54: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 702 op/s 2024-04-12T21:49:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:12 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3981741676' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:49:12.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 129986 12.26 MB/sec execute 112 sec latency 73.943 ms 2024-04-12T21:49:13.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 130315 12.24 MB/sec execute 113 sec latency 40.270 ms 2024-04-12T21:49:14.231 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:14 smithi123 ceph-mon[24842]: pgmap v55: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 452 op/s 2024-04-12T21:49:14.506 INFO:teuthology.orchestra.run.smithi123.stdout: 5 130736 12.30 MB/sec execute 114 sec latency 35.981 ms 2024-04-12T21:49:14.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:14 smithi177 ceph-mon[26903]: pgmap v55: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.5 MiB/s wr, 452 op/s 2024-04-12T21:49:15.248 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:15 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:15.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 131129 12.34 MB/sec execute 115 sec latency 41.929 ms 2024-04-12T21:49:15.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:15 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:16.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: pgmap v56: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 492 op/s 2024-04-12T21:49:16.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:16.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:16.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: Standby manager daemon smithi123.psjrfs restarted 2024-04-12T21:49:16.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: Standby manager daemon smithi123.psjrfs started 2024-04-12T21:49:16.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.psjrfs/crt"}]: dispatch 2024-04-12T21:49:16.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:49:16.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.psjrfs/key"}]: dispatch 2024-04-12T21:49:16.270 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:16 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:49:16.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 131463 12.32 MB/sec execute 116 sec latency 47.580 ms 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: pgmap v56: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 492 op/s 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: Standby manager daemon smithi123.psjrfs restarted 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: Standby manager daemon smithi123.psjrfs started 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.psjrfs/crt"}]: dispatch 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.psjrfs/key"}]: dispatch 2024-04-12T21:49:16.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:16 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:49:17.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 131859 12.27 MB/sec execute 117 sec latency 83.320 ms 2024-04-12T21:49:17.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:17 smithi123 ceph-mon[24842]: mgrmap e29: smithi177.whybza(active, since 108s), standbys: smithi123.psjrfs 2024-04-12T21:49:17.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:17 smithi123 ceph-mon[24842]: pgmap v57: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-04-12T21:49:18.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:17 smithi177 ceph-mon[26903]: mgrmap e29: smithi177.whybza(active, since 108s), standbys: smithi123.psjrfs 2024-04-12T21:49:18.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:17 smithi177 ceph-mon[26903]: pgmap v57: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-04-12T21:49:18.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 132192 12.20 MB/sec execute 118 sec latency 37.317 ms 2024-04-12T21:49:19.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 132458 12.16 MB/sec execute 119 sec latency 52.052 ms 2024-04-12T21:49:19.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:19 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:19.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:19 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:19.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:19 smithi123 ceph-mon[24842]: pgmap v58: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 365 op/s 2024-04-12T21:49:20.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:19 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:20.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:19 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:20.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:19 smithi177 ceph-mon[26903]: pgmap v58: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 365 op/s 2024-04-12T21:49:20.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 132791 12.13 MB/sec execute 120 sec latency 60.003 ms 2024-04-12T21:49:21.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 133150 12.08 MB/sec execute 121 sec latency 61.694 ms 2024-04-12T21:49:22.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:22 smithi123 ceph-mon[24842]: pgmap v59: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-04-12T21:49:22.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 133540 12.04 MB/sec execute 122 sec latency 61.294 ms 2024-04-12T21:49:22.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:22 smithi177 ceph-mon[26903]: pgmap v59: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-04-12T21:49:23.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 134020 12.08 MB/sec execute 123 sec latency 30.307 ms 2024-04-12T21:49:24.167 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:24 smithi123 ceph-mon[24842]: pgmap v60: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 412 op/s 2024-04-12T21:49:24.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:24 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:24.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:24 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:24.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:24 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:49:24.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:24 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:49:24.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:24 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:24.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:24 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:24.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 134436 12.11 MB/sec execute 124 sec latency 41.632 ms 2024-04-12T21:49:24.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:24 smithi177 ceph-mon[26903]: pgmap v60: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 412 op/s 2024-04-12T21:49:24.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:24 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:24.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:24 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:24.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:24 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:49:24.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:24 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:49:24.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:24 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:24.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:24 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: pgmap v61: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: Upgrade: Need to upgrade myself (mgr.smithi177.whybza) 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: Upgrade: Need to upgrade myself (mgr.smithi177.whybza) 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: Failing over to other MGR 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr fail", "who": "smithi177.whybza"}]: dispatch 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr fail", "who": "smithi177.whybza"}]: dispatch 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: Standby manager daemon smithi177.whybza started 2024-04-12T21:49:25.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:25 smithi123 ceph-mon[24842]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: pgmap v61: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: Upgrade: Need to upgrade myself (mgr.smithi177.whybza) 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: Upgrade: Need to upgrade myself (mgr.smithi177.whybza) 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: Failing over to other MGR 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: from='mgr.24417 172.21.15.177:0/3166868617' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr fail", "who": "smithi177.whybza"}]: dispatch 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd=[{"prefix": "mgr fail", "who": "smithi177.whybza"}]: dispatch 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza started 2024-04-12T21:49:25.473 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:25 smithi177 ceph-mon[26903]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T21:49:25.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 134909 12.17 MB/sec execute 125 sec latency 31.236 ms 2024-04-12T21:49:26.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd='[{"prefix": "mgr fail", "who": "smithi177.whybza"}]': finished 2024-04-12T21:49:26.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: mgrmap e30: smithi123.psjrfs(active, starting, since 1.0007s), standbys: smithi177.whybza 2024-04-12T21:49:26.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:49:26.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:49:26.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi177.whybza", "id": "smithi177.whybza"}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:49:26.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:49:26.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:49:26.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:49:26.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:49:26.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:26.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:26.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:49:26.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:26 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:49:26.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 135320 12.15 MB/sec execute 126 sec latency 36.111 ms 2024-04-12T21:49:26.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.24417 ' entity='mgr.smithi177.whybza' cmd='[{"prefix": "mgr fail", "who": "smithi177.whybza"}]': finished 2024-04-12T21:49:26.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: mgrmap e30: smithi123.psjrfs(active, starting, since 1.0007s), standbys: smithi177.whybza 2024-04-12T21:49:26.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:49:26.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi177.whybza", "id": "smithi177.whybza"}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:49:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:49:26.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:49:26.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:49:26.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:49:26.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:26.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:26.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:49:26.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:26 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:49:27.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:27 smithi123 ceph-mon[24842]: [12/Apr/2024:21:49:25] ENGINE Bus STARTING 2024-04-12T21:49:27.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:27 smithi123 ceph-mon[24842]: [12/Apr/2024:21:49:25] ENGINE Serving on https://172.21.15.123:7150 2024-04-12T21:49:27.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:27 smithi123 ceph-mon[24842]: [12/Apr/2024:21:49:26] ENGINE Serving on http://172.21.15.123:8765 2024-04-12T21:49:27.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:27 smithi123 ceph-mon[24842]: [12/Apr/2024:21:49:26] ENGINE Bus STARTED 2024-04-12T21:49:27.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:27 smithi123 ceph-mon[24842]: mgrmap e31: smithi123.psjrfs(active, since 2s), standbys: smithi177.whybza 2024-04-12T21:49:27.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:27 smithi123 ceph-mon[24842]: pgmap v3: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:49:27.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:27 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:27.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:27 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:27.409 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:27 smithi177 ceph-mon[26903]: [12/Apr/2024:21:49:25] ENGINE Bus STARTING 2024-04-12T21:49:27.409 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:27 smithi177 ceph-mon[26903]: [12/Apr/2024:21:49:25] ENGINE Serving on https://172.21.15.123:7150 2024-04-12T21:49:27.410 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:27 smithi177 ceph-mon[26903]: [12/Apr/2024:21:49:26] ENGINE Serving on http://172.21.15.123:8765 2024-04-12T21:49:27.410 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:27 smithi177 ceph-mon[26903]: [12/Apr/2024:21:49:26] ENGINE Bus STARTED 2024-04-12T21:49:27.410 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:27 smithi177 ceph-mon[26903]: mgrmap e31: smithi123.psjrfs(active, since 2s), standbys: smithi177.whybza 2024-04-12T21:49:27.410 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:27 smithi177 ceph-mon[26903]: pgmap v3: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:49:27.410 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:27 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:27.410 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:27 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:27.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 135552 12.08 MB/sec execute 127 sec latency 48.584 ms 2024-04-12T21:49:28.234 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:28 smithi123 ceph-mon[24842]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T21:49:28.234 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:28 smithi123 ceph-mon[24842]: Cluster is now healthy 2024-04-12T21:49:28.234 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:28 smithi123 ceph-mon[24842]: pgmap v4: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:49:28.451 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:28 smithi177 ceph-mon[26903]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T21:49:28.452 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:28 smithi177 ceph-mon[26903]: Cluster is now healthy 2024-04-12T21:49:28.452 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:28 smithi177 ceph-mon[26903]: pgmap v4: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:49:28.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 135766 12.01 MB/sec execute 128 sec latency 66.042 ms 2024-04-12T21:49:29.273 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:29 smithi123 ceph-mon[24842]: mgrmap e32: smithi123.psjrfs(active, since 4s), standbys: smithi177.whybza 2024-04-12T21:49:29.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 135964 11.95 MB/sec execute 129 sec latency 77.580 ms 2024-04-12T21:49:29.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:29 smithi177 ceph-mon[26903]: mgrmap e32: smithi123.psjrfs(active, since 4s), standbys: smithi177.whybza 2024-04-12T21:49:30.200 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:30 smithi177 ceph-mon[26903]: pgmap v5: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:49:30.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:30 smithi123 ceph-mon[24842]: pgmap v5: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:49:30.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 136220 11.92 MB/sec execute 130 sec latency 91.351 ms 2024-04-12T21:49:31.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 136487 11.85 MB/sec execute 131 sec latency 150.071 ms 2024-04-12T21:49:32.410 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:32 smithi123 ceph-mon[24842]: pgmap v6: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:49:32.410 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:32 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:32.410 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:32 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:32.440 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:32 smithi177 ceph-mon[26903]: pgmap v6: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:49:32.441 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:32 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:32.441 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:32 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:32.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 136912 11.84 MB/sec execute 132 sec latency 50.518 ms 2024-04-12T21:49:33.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 137312 11.82 MB/sec execute 133 sec latency 43.653 ms 2024-04-12T21:49:34.216 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:34 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:34.217 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:34 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:34.217 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:34 smithi123 ceph-mon[24842]: pgmap v7: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 2.2 MiB/s wr, 305 op/s 2024-04-12T21:49:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:34 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:34 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:34 smithi177 ceph-mon[26903]: pgmap v7: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 2.2 MiB/s wr, 305 op/s 2024-04-12T21:49:34.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 137656 11.83 MB/sec execute 134 sec latency 41.690 ms 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: Standby manager daemon smithi177.whybza restarted 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: Standby manager daemon smithi177.whybza started 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/1445561322' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/1445561322' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/1445561322' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/1445561322' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: pgmap v8: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 1.7 MiB/s wr, 237 op/s 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:35.487 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:35 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:35.509 INFO:teuthology.orchestra.run.smithi123.stdout: 5 138048 11.86 MB/sec execute 135 sec latency 43.816 ms 2024-04-12T21:49:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza restarted 2024-04-12T21:49:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza started 2024-04-12T21:49:35.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/1445561322' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:49:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/1445561322' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:49:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/1445561322' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:49:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/1445561322' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:49:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: pgmap v8: 97 pgs: 97 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 1.7 MiB/s wr, 237 op/s 2024-04-12T21:49:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:35 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:36.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 138429 11.88 MB/sec execute 136 sec latency 63.790 ms 2024-04-12T21:49:36.525 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:36 smithi123 ceph-mon[24842]: mgrmap e33: smithi123.psjrfs(active, since 11s), standbys: smithi177.whybza 2024-04-12T21:49:36.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:36 smithi177 ceph-mon[26903]: mgrmap e33: smithi123.psjrfs(active, since 11s), standbys: smithi177.whybza 2024-04-12T21:49:37.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 138748 11.86 MB/sec execute 137 sec latency 38.504 ms 2024-04-12T21:49:37.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:37 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:37.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:37 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:37.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:37 smithi123 ceph-mon[24842]: pgmap v9: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 431 op/s 2024-04-12T21:49:38.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:37 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:38.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:37 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:38.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:37 smithi177 ceph-mon[26903]: pgmap v9: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 431 op/s 2024-04-12T21:49:38.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 139056 11.81 MB/sec execute 138 sec latency 51.228 ms 2024-04-12T21:49:39.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 139364 11.77 MB/sec execute 139 sec latency 43.507 ms 2024-04-12T21:49:40.338 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:40 smithi123 ceph-mon[24842]: pgmap v10: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 3.7 MiB/s wr, 394 op/s 2024-04-12T21:49:40.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 139703 11.75 MB/sec execute 140 sec latency 50.213 ms 2024-04-12T21:49:40.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:40 smithi177 ceph-mon[26903]: pgmap v10: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 3.7 MiB/s wr, 394 op/s 2024-04-12T21:49:41.353 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:41 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:41.353 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:41 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:41.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 140279 11.74 MB/sec execute 141 sec latency 32.067 ms 2024-04-12T21:49:41.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:41 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:41.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:41 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:41.904 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:49:42.358 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:42 smithi123 ceph-mon[24842]: pgmap v11: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 3.7 MiB/s wr, 394 op/s 2024-04-12T21:49:42.359 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:42 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:42.359 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:42 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:42.359 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:42 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:49:42.359 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:42 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:49:42.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 140825 11.74 MB/sec execute 142 sec latency 21.404 ms 2024-04-12T21:49:42.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:42 smithi177 ceph-mon[26903]: pgmap v11: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 3.7 MiB/s wr, 394 op/s 2024-04-12T21:49:42.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:42 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:42.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:42 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:42.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:42 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:49:42.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:42 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (101s) 7s ago 10m 25.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (10m) 7s ago 10m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (10m) 11s ago 10m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 unknown 7s ago 10m - - dad864ee21e9 e38e85ea21ca 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (4m) 7s ago 4m 37.4M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (4m) 11s ago 4m 13.8M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (37s) 7s ago 11m 479M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283 running (2m) 11s ago 10m 110M - 19.0.0-2053-gb277feb2 668895944f0e 344eee5ab888 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (11m) 7s ago 12m 53.0M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (10m) 11s ago 10m 45.7M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (4m) 7s ago 4m 252M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:49:43.091 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (4m) 11s ago 4m 82.4M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (76s) 7s ago 10m 19.3M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (46s) 11s ago 9m 9042k - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (9m) 7s ago 9m 479M 1936M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (8m) 7s ago 8m 403M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (8m) 7s ago 8m 285M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (8m) 7s ago 7m 436M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (7m) 11s ago 7m 466M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (7m) 11s ago 7m 514M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (6m) 11s ago 6m 430M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (6m) 11s ago 6m 346M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:49:43.092 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (51s) 7s ago 9m 51.5M - 2.33.4 514e6a882f6e 1758e495ce6b 2024-04-12T21:49:43.371 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:43 smithi123 ceph-mon[24842]: from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:43.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 141299 11.78 MB/sec execute 143 sec latency 77.909 ms 2024-04-12T21:49:43.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:43 smithi123 ceph-mon[24842]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:49:43.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:43 smithi123 ceph-mon[24842]: Updating smithi177:/etc/ceph/ceph.conf 2024-04-12T21:49:43.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:43 smithi123 ceph-mon[24842]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:49:43.736 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:49:43.737 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:49:43.737 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:49:43.737 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-12T21:49:43.737 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:49:43.737 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:49:43.737 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:49:43.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:43 smithi177 ceph-mon[26903]: from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:43.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:43 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:49:43.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:43 smithi177 ceph-mon[26903]: Updating smithi177:/etc/ceph/ceph.conf 2024-04-12T21:49:43.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:43 smithi177 ceph-mon[26903]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:49:44.322 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:49:44.322 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:49:44.322 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:49:44.323 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:49:44.323 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:49:44.323 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr" 2024-04-12T21:49:44.323 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:49:44.323 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "2/23 daemons upgraded", 2024-04-12T21:49:44.323 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-04-12T21:49:44.323 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:49:44.323 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:49:44.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 141887 11.86 MB/sec execute 144 sec latency 28.586 ms 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='client.24563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: pgmap v12: 97 pgs: 97 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 5.8 MiB/s wr, 621 op/s 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.633 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.634 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/800040193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:49:44.634 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.634 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:44 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='client.24563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:44.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:49:44.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:49:44.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:49:44.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:49:44.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: pgmap v12: 97 pgs: 97 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 5.8 MiB/s wr, 621 op/s 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/800040193' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:44 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:44.928 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:49:44.929 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:49:44.929 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:49:45.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 142329 11.87 MB/sec execute 145 sec latency 54.150 ms 2024-04-12T21:49:45.641 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:45 smithi123 ceph-mon[24842]: pgmap v13: 97 pgs: 97 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-12T21:49:45.641 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:45 smithi123 ceph-mon[24842]: Reconfiguring prometheus.smithi123 (dependencies changed)... 2024-04-12T21:49:45.641 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:45 smithi123 ceph-mon[24842]: Reconfiguring daemon prometheus.smithi123 on smithi123 2024-04-12T21:49:45.641 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:45 smithi123 ceph-mon[24842]: from='client.24575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:45.641 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:45 smithi123 ceph-mon[24842]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T21:49:45.642 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:45 smithi123 ceph-mon[24842]: from='client.? 172.21.15.123:0/3593530072' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:49:45.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:45 smithi177 ceph-mon[26903]: pgmap v13: 97 pgs: 97 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-12T21:49:45.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:45 smithi177 ceph-mon[26903]: Reconfiguring prometheus.smithi123 (dependencies changed)... 2024-04-12T21:49:45.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:45 smithi177 ceph-mon[26903]: Reconfiguring daemon prometheus.smithi123 on smithi123 2024-04-12T21:49:45.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:45 smithi177 ceph-mon[26903]: from='client.24575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:49:45.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:45 smithi177 ceph-mon[26903]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T21:49:45.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:45 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3593530072' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:49:46.511 INFO:teuthology.orchestra.run.smithi123.stdout: 5 142811 11.84 MB/sec execute 146 sec latency 45.216 ms 2024-04-12T21:49:47.405 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:47 smithi123 ceph-mon[24842]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-12T21:49:47.510 INFO:teuthology.orchestra.run.smithi123.stdout: 5 143365 11.85 MB/sec execute 147 sec latency 36.728 ms 2024-04-12T21:49:47.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:47 smithi177 ceph-mon[26903]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-12T21:49:48.514 INFO:teuthology.orchestra.run.smithi123.stdout: 5 143916 11.84 MB/sec execute 148 sec latency 49.366 ms 2024-04-12T21:49:49.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:49 smithi123 ceph-mon[24842]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.2 MiB/s wr, 596 op/s 2024-04-12T21:49:49.514 INFO:teuthology.orchestra.run.smithi123.stdout: 5 144476 11.84 MB/sec execute 149 sec latency 41.584 ms 2024-04-12T21:49:49.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:49 smithi177 ceph-mon[26903]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.2 MiB/s wr, 596 op/s 2024-04-12T21:49:50.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 145074 11.92 MB/sec execute 150 sec latency 36.858 ms 2024-04-12T21:49:50.700 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:50 smithi123 ceph-mon[24842]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.2 MiB/s wr, 596 op/s 2024-04-12T21:49:50.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:50 smithi177 ceph-mon[26903]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.2 MiB/s wr, 596 op/s 2024-04-12T21:49:51.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 145550 11.97 MB/sec execute 151 sec latency 40.379 ms 2024-04-12T21:49:52.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 145958 11.96 MB/sec execute 152 sec latency 28.943 ms 2024-04-12T21:49:52.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:52 smithi123 ceph-mon[24842]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 969 op/s 2024-04-12T21:49:53.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:52 smithi177 ceph-mon[26903]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 969 op/s 2024-04-12T21:49:53.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 146388 11.93 MB/sec execute 153 sec latency 32.497 ms 2024-04-12T21:49:54.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:54 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:54.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:54 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:54.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:54 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:49:54.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:54 smithi123 ceph-mon[24842]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:49:54.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:54 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:49:54.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:54 smithi123 ceph-mon[24842]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.6 MiB/s wr, 711 op/s 2024-04-12T21:49:54.514 INFO:teuthology.orchestra.run.smithi123.stdout: 5 146791 11.90 MB/sec execute 154 sec latency 43.779 ms 2024-04-12T21:49:54.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:54 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:54.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:54 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:54.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:54 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:49:54.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:54 smithi177 ceph-mon[26903]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:49:54.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:54 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:49:54.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:54 smithi177 ceph-mon[26903]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.6 MiB/s wr, 711 op/s 2024-04-12T21:49:55.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 147351 11.92 MB/sec execute 155 sec latency 27.683 ms 2024-04-12T21:49:56.414 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:56 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:56.414 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:56 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:56.414 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:56 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:49:56.414 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:56 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:49:56.414 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:56 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:49:56.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 147873 11.90 MB/sec execute 156 sec latency 55.511 ms 2024-04-12T21:49:56.587 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:56 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:49:56.587 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:56 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:49:56.587 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:56 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:49:56.587 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:56 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:49:56.588 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:56 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:49:57.313 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:57 smithi123 ceph-mon[24842]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 626 op/s 2024-04-12T21:49:57.314 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:57 smithi123 ceph-mon[24842]: Upgrade: Updating mgr.smithi177.whybza 2024-04-12T21:49:57.314 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:57 smithi123 ceph-mon[24842]: Deploying daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:49:57.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 148284 11.93 MB/sec execute 157 sec latency 47.995 ms 2024-04-12T21:49:57.682 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:57 smithi177 ceph-mon[26903]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.7 MiB/s wr, 626 op/s 2024-04-12T21:49:57.682 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:57 smithi177 ceph-mon[26903]: Upgrade: Updating mgr.smithi177.whybza 2024-04-12T21:49:57.682 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:57 smithi177 ceph-mon[26903]: Deploying daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:49:58.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 148658 11.94 MB/sec execute 158 sec latency 56.126 ms 2024-04-12T21:49:59.369 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:49:59 smithi123 ceph-mon[24842]: pgmap v20: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 914 op/s 2024-04-12T21:49:59.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 148989 11.95 MB/sec execute 159 sec latency 47.601 ms 2024-04-12T21:49:59.763 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:49:59 smithi177 ceph-mon[26903]: pgmap v20: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 914 op/s 2024-04-12T21:50:00.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 149272 11.94 MB/sec execute 160 sec latency 56.270 ms 2024-04-12T21:50:00.642 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:00 smithi123 ceph-mon[24842]: pgmap v21: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 616 op/s 2024-04-12T21:50:00.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:00 smithi123 ceph-mon[24842]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:00.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:00 smithi123 ceph-mon[24842]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:00.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:00 smithi123 ceph-mon[24842]: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:00.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:00 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:00.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:00 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:00.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:00 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:00.762 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:00 smithi177 ceph-mon[26903]: pgmap v21: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 616 op/s 2024-04-12T21:50:00.763 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:00 smithi177 ceph-mon[26903]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:00.763 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:00 smithi177 ceph-mon[26903]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:00.763 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:00 smithi177 ceph-mon[26903]: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:00.763 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:00 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:00.763 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:00 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:00.764 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:00 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:01.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 149595 11.89 MB/sec execute 161 sec latency 42.612 ms 2024-04-12T21:50:02.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 149986 11.86 MB/sec execute 162 sec latency 29.407 ms 2024-04-12T21:50:02.914 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:02 smithi123 ceph-mon[24842]: pgmap v22: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 828 op/s 2024-04-12T21:50:03.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:02 smithi177 ceph-mon[26903]: pgmap v22: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 828 op/s 2024-04-12T21:50:03.515 INFO:teuthology.orchestra.run.smithi123.stdout: 5 150356 11.84 MB/sec execute 163 sec latency 67.304 ms 2024-04-12T21:50:04.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 150794 11.84 MB/sec execute 164 sec latency 32.560 ms 2024-04-12T21:50:04.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:04 smithi123 ceph-mon[24842]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-04-12T21:50:04.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:04 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:04.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:04 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:05.242 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:04 smithi177 ceph-mon[26903]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-04-12T21:50:05.242 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:04 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:05.242 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:04 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:05.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 151260 11.81 MB/sec execute 165 sec latency 62.392 ms 2024-04-12T21:50:06.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 151613 11.79 MB/sec execute 166 sec latency 48.397 ms 2024-04-12T21:50:07.014 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:06 smithi123 ceph-mon[24842]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-04-12T21:50:07.014 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:06 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:07.014 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:06 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:07.267 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:06 smithi177 ceph-mon[26903]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-04-12T21:50:07.268 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:06 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:07.268 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:06 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:07.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 152044 11.83 MB/sec execute 167 sec latency 37.179 ms 2024-04-12T21:50:08.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 152403 11.85 MB/sec execute 168 sec latency 40.250 ms 2024-04-12T21:50:09.070 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:08 smithi123 ceph-mon[24842]: pgmap v25: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 757 op/s 2024-04-12T21:50:09.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:08 smithi123 ceph-mon[24842]: Standby manager daemon smithi177.whybza restarted 2024-04-12T21:50:09.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:08 smithi123 ceph-mon[24842]: Standby manager daemon smithi177.whybza started 2024-04-12T21:50:09.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:08 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/3714367777' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:50:09.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:08 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/3714367777' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:50:09.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:08 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/3714367777' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:50:09.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:08 smithi123 ceph-mon[24842]: from='mgr.? 172.21.15.177:0/3714367777' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:50:09.258 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:08 smithi177 ceph-mon[26903]: pgmap v25: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 757 op/s 2024-04-12T21:50:09.258 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:08 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza restarted 2024-04-12T21:50:09.258 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:08 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza started 2024-04-12T21:50:09.258 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:08 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/3714367777' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:50:09.258 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:08 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/3714367777' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:50:09.258 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:08 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/3714367777' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:50:09.258 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:08 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/3714367777' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:50:09.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 152689 11.84 MB/sec execute 169 sec latency 59.630 ms 2024-04-12T21:50:10.111 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:09 smithi123 ceph-mon[24842]: mgrmap e34: smithi123.psjrfs(active, since 44s), standbys: smithi177.whybza 2024-04-12T21:50:10.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:09 smithi177 ceph-mon[26903]: mgrmap e34: smithi123.psjrfs(active, since 44s), standbys: smithi177.whybza 2024-04-12T21:50:10.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 152930 11.81 MB/sec execute 170 sec latency 67.239 ms 2024-04-12T21:50:11.134 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: pgmap v26: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 470 op/s 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.135 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:10 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: pgmap v26: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 470 op/s 2024-04-12T21:50:11.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:50:11.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:50:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:50:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:10 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:11.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 153122 11.75 MB/sec execute 171 sec latency 53.283 ms 2024-04-12T21:50:12.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: pgmap v27: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-12T21:50:12.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:12.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:50:12.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: Upgrade: Setting container_image for all mgr 2024-04-12T21:50:12.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:12.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi123.psjrfs"}]: dispatch 2024-04-12T21:50:12.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi123.psjrfs"}]': finished 2024-04-12T21:50:12.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi177.whybza"}]: dispatch 2024-04-12T21:50:12.162 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi177.whybza"}]': finished 2024-04-12T21:50:12.162 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:11 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T21:50:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: pgmap v27: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-04-12T21:50:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:50:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: Upgrade: Setting container_image for all mgr 2024-04-12T21:50:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:12.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi123.psjrfs"}]: dispatch 2024-04-12T21:50:12.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi123.psjrfs"}]': finished 2024-04-12T21:50:12.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi177.whybza"}]: dispatch 2024-04-12T21:50:12.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi177.whybza"}]': finished 2024-04-12T21:50:12.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:12 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T21:50:12.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 153396 11.70 MB/sec execute 172 sec latency 50.387 ms 2024-04-12T21:50:13.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 153769 11.68 MB/sec execute 173 sec latency 41.376 ms 2024-04-12T21:50:14.431 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:14 smithi123 ceph-mon[24842]: pgmap v28: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-04-12T21:50:14.431 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:14 smithi123 ceph-mon[24842]: Upgrade: Updating mon.smithi123 2024-04-12T21:50:14.431 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:14 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:14.431 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:14 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:50:14.431 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:14 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:50:14.431 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:14 smithi123 ceph-mon[24842]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:50:14.431 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:14 smithi123 ceph-mon[24842]: Deploying daemon mon.smithi123 on smithi123 2024-04-12T21:50:14.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 154316 11.71 MB/sec execute 174 sec latency 31.073 ms 2024-04-12T21:50:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:14 smithi177 ceph-mon[26903]: pgmap v28: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-04-12T21:50:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:14 smithi177 ceph-mon[26903]: Upgrade: Updating mon.smithi123 2024-04-12T21:50:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:14 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:14.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:14 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:50:14.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:14 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:50:14.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:14 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:50:14.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:14 smithi177 ceph-mon[26903]: Deploying daemon mon.smithi123 on smithi123 2024-04-12T21:50:15.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 154875 11.69 MB/sec execute 175 sec latency 56.993 ms 2024-04-12T21:50:15.558 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:50:15.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:15 smithi123 ceph-mon[24842]: pgmap v29: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-04-12T21:50:15.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:15 smithi177 ceph-mon[26903]: pgmap v29: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-04-12T21:50:16.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:16 smithi123 systemd[1]: Stopping Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:50:16.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 155279 11.69 MB/sec execute 176 sec latency 67.829 ms 2024-04-12T21:50:16.709 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:16 smithi123 ceph-mon[24842]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:50:16.709 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:16 smithi123 ceph-mon[24842]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:50:16.740 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:50:16.740 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (2m) 5s ago 11m 25.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:50:16.740 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (11m) 5s ago 11m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (10m) 12s ago 10m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 unknown 5s ago 11m - - 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (5m) 5s ago 5m 37.8M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (5m) 12s ago 5m 14.0M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (70s) 5s ago 12m 494M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (17s) 12s ago 10m 64.0M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (12m) 5s ago 12m 52.8M 2048M 17.2.0 e1d6a67b021e 8e8322580819 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (10m) 12s ago 10m 46.4M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (5m) 5s ago 5m 260M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (5m) 12s ago 5m 84.2M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (110s) 5s ago 11m 22.0M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (80s) 12s ago 10m 19.0M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (10m) 5s ago 9m 505M 1936M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (9m) 5s ago 9m 444M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (9m) 5s ago 9m 320M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:50:16.741 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (8m) 5s ago 8m 477M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:50:16.742 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (8m) 12s ago 8m 491M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:50:16.742 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (7m) 12s ago 7m 311M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:50:16.742 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (7m) 12s ago 7m 493M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:50:16.742 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (6m) 12s ago 6m 386M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:50:16.742 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (24s) 5s ago 10m 48.8M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:50:16.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:16 smithi177 ceph-mon[26903]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:50:16.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:16 smithi177 ceph-mon[26903]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:50:16.964 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:16 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[24838]: 2024-04-12T21:50:16.775+0000 7ff2ba42b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi123 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:50:16.964 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:16 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[24838]: 2024-04-12T21:50:16.775+0000 7ff2ba42b700 -1 mon.smithi123@0(leader) e2 *** Got Signal Terminated *** 2024-04-12T21:50:16.964 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:16 smithi123 podman[66990]: 2024-04-12 21:50:16.806795824 +0000 UTC m=+0.219265027 container died 8e8322580819e87f2a5f1b3e8b359afbce14d7277f3f1011ff6143a8d262a6b1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, version=8, release=754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-type=git, GIT_BRANCH=HEAD) 2024-04-12T21:50:17.437 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:50:17.437 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:50:17.437 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:50:17.437 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:50:17.437 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:50:17.438 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:50:17.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 155730 11.74 MB/sec execute 177 sec latency 44.618 ms 2024-04-12T21:50:17.718 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:17 smithi123 podman[66990]: 2024-04-12 21:50:17.544782544 +0000 UTC m=+0.957251749 container cleanup 8e8322580819e87f2a5f1b3e8b359afbce14d7277f3f1011ff6143a8d262a6b1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, ceph=True, maintainer=Guillaume Abrioux , vcs-type=git, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.component=centos-stream-container, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-12T21:50:17.719 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:17 smithi123 bash[66990]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123 2024-04-12T21:50:17.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:17 smithi123 podman[67005]: 2024-04-12 21:50:17.844551149 +0000 UTC m=+1.035833441 container remove 8e8322580819e87f2a5f1b3e8b359afbce14d7277f3f1011ff6143a8d262a6b1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, GIT_BRANCH=HEAD, release=754, architecture=x86_64, name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-type=git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, ceph=True, GIT_CLEAN=True, distribution-scope=public, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.buildah.version=1.19.8) 2024-04-12T21:50:17.993 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr" 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "2/23 daemons upgraded", 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mon daemons", 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:50:17.994 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:50:18.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:18 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123.service: Deactivated successfully. 2024-04-12T21:50:18.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:18 smithi123 systemd[1]: Stopped Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:50:18.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:18 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123.service: Consumed 16.612s CPU time. 2024-04-12T21:50:18.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 156128 11.76 MB/sec execute 178 sec latency 37.900 ms 2024-04-12T21:50:18.660 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:18.661 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:18.661 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:18.724 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:18 smithi123 systemd[1]: Starting Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:50:18.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:18 smithi123 podman[67207]: 2024-04-12 21:50:18.85443406 +0000 UTC m=+0.028714534 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:50:19.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 podman[67207]: 2024-04-12 21:50:19.104044407 +0000 UTC m=+0.278324905 container create ef4cffbb2b460eb2e393f735c33f4dc979238120664d0918f60d1d7ef52bb3ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True) 2024-04-12T21:50:19.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 156463 11.75 MB/sec execute 179 sec latency 44.286 ms 2024-04-12T21:50:19.729 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 podman[67207]: 2024-04-12 21:50:19.506034327 +0000 UTC m=+0.680314778 container init ef4cffbb2b460eb2e393f735c33f4dc979238120664d0918f60d1d7ef52bb3ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, RELEASE=squid-b277feb) 2024-04-12T21:50:19.729 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 podman[67207]: 2024-04-12 21:50:19.520217561 +0000 UTC m=+0.694498012 container start ef4cffbb2b460eb2e393f735c33f4dc979238120664d0918f60d1d7ef52bb3ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T21:50:19.729 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev), process ceph-mon, pid 2 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: pidfile_write: ignore empty --pid-file 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: load: jerasure load: lrc 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: RocksDB version: 7.9.2 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Git sha 0 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Compile date 2024-04-09 21:18:54 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: DB SUMMARY 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: DB Session ID: 4Y0BQYVM0ODIET9MQMYG 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: CURRENT file: CURRENT 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-04-12T21:50:19.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi123/store.db dir, Total Num: 1, files: 000030.sst 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi123/store.db: 000028.log size: 2467000 ; 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.error_if_exists: 0 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.create_if_missing: 0 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.paranoid_checks: 1 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.env: 0x561e9c521c60 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.info_log: 0x561e9ed379a0 2024-04-12T21:50:19.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T21:50:19.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.statistics: (nil) 2024-04-12T21:50:19.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.use_fsync: 0 2024-04-12T21:50:19.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_log_file_size: 0 2024-04-12T21:50:19.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T21:50:19.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T21:50:19.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T21:50:19.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.allow_fallocate: 1 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.use_direct_reads: 0 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.db_log_dir: 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.wal_dir: 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T21:50:19.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.write_buffer_manager: 0x561e9ed0a1e0 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.unordered_write: 0 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T21:50:19.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.row_cache: None 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.wal_filter: None 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.two_write_queues: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.wal_compression: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.atomic_flush: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.log_readahead_size: 0 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T21:50:19.736 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_background_jobs: 2 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_background_compactions: -1 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_subcompactions: 1 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T21:50:19.737 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_open_files: -1 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_background_flushes: -1 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Compression algorithms supported: 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: kZSTD supported: 0 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: kXpressCompression supported: 0 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: kZlibCompression supported: 1 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: kSnappyCompression supported: 1 2024-04-12T21:50:19.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: kLZ4Compression supported: 1 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: kBZip2Compression supported: 0 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi123/store.db/MANIFEST-000015 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.merge_operator: 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_filter: None 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561e9ed37aa0) 2024-04-12T21:50:19.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: cache_index_and_filter_blocks: 1 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: pin_top_level_index_and_filter: 1 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_type: 0 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_block_index_type: 0 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_shortening: 1 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: checksum: 4 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: no_block_cache: 0 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache: 0x561e9ed3b090 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_name: BinnedLRUCache 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_options: 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: capacity : 536870912 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: num_shard_bits : 4 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: strict_capacity_limit : 0 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: high_pri_pool_ratio: 0.000 2024-04-12T21:50:19.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_compressed: (nil) 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: persistent_cache: (nil) 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_size: 4096 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_size_deviation: 10 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_restart_interval: 16 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_block_restart_interval: 1 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: metadata_block_size: 4096 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: partition_filters: 0 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: use_delta_encoding: 1 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: filter_policy: bloomfilter 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: whole_key_filtering: 1 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: verify_compression: 0 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: read_amp_bytes_per_bit: 0 2024-04-12T21:50:19.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: format_version: 5 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: enable_index_compression: 1 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_align: 0 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: max_auto_readahead_size: 262144 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: prepopulate_block_cache: 0 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: initial_auto_readahead_size: 8192 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression: NoCompression 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.num_levels: 7 2024-04-12T21:50:19.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T21:50:19.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T21:50:19.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T21:50:19.745 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T21:50:19.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.table_properties_collectors: 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.inplace_update_support: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.bloom_locality: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.max_successive_merges: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T21:50:19.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.ttl: 2592000 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.enable_blob_files: false 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.min_blob_size: 0 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T21:50:19.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi123/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11837, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f961ae73-0892-4bc7-b66d-c52f9795baea 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712958619564606, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712958619574273, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2279352, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11828, "largest_seqno": 12885, "table_properties": {"data_size": 2274027, "index_size": 3154, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10714, "raw_average_key_size": 23, "raw_value_size": 2263944, "raw_average_value_size": 4921, "num_data_blocks": 144, "num_entries": 460, "num_filter_entries": 460, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712958619, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f961ae73-0892-4bc7-b66d-c52f9795baea", "db_session_id": "4Y0BQYVM0ODIET9MQMYG", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712958619574355, "job": 1, "event": "recovery_finished"} 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi123/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561e9ee2c000 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: DB pointer 0x561e9ee16000 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** DB Stats ** 2024-04-12T21:50:19.749 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: L0 1/0 2.17 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 468.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: L6 1/0 6.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Sum 2/0 8.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 468.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-12T21:50:19.750 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 468.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 468.7 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T21:50:19.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative compaction: 0.00 GB write, 131.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval compaction: 0.00 GB write, 131.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Block cache BinnedLRUCache@0x561e9ed3b090#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: starting mon.smithi123 rank 0 at public addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] at bind addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi123 fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123@-1(???) e2 preinit fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123@-1(???).mds e6 new map 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123@-1(???).mds e6 print_map 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: e6 2024-04-12T21:50:19.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: legacy client fscid: 1 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Filesystem 'foofs' (1) 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: fs_name foofs 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: epoch 5 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: created 2024-04-12T21:44:41.570840+0000 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: modified 2024-04-12T21:44:47.590655+0000 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: tableserver 0 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: root 0 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: session_timeout 60 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: session_autoclose 300 2024-04-12T21:50:19.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: max_file_size 1099511627776 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: max_xattr_size 65536 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: required_client_features {} 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: last_failure 0 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: last_failure_osd_epoch 0 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: max_mds 1 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: in 0 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: up {0=14486} 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: failed 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: damaged 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: stopped 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_pools [3] 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: metadata_pool 2 2024-04-12T21:50:19.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: inline_data disabled 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: balancer 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: bal_rank_mask -1 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: standby_count_wanted 1 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: qdb_cluster leader: 0 members: 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: [mds.foofs.smithi123.poxexz{0:14486} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.123:6834/1104424315,v1:172.21.15.123:6835/1104424315] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Standby daemons: 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: [mds.foofs.smithi177.hqctzz{-1:24269} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T21:50:19.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-04-12T21:50:19.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:50:19.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:50:19.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: pgmap v30: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 751 op/s 2024-04-12T21:50:19.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/2001273971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:50:19.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:50:19.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3776024732' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: pgmap v31: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123 calling monitor election 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mon.smithi123 is new leader, mons smithi123,smithi177 in quorum (ranks 0,1) 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: monmap epoch 2 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: last_changed 2024-04-12T21:39:39.773252+0000 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: created 2024-04-12T21:37:26.292641+0000 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: min_mon_release 17 (quincy) 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: election_strategy: 1 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.smithi177 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T21:50:19.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mgrmap e34: smithi123.psjrfs(active, since 55s), standbys: smithi177.whybza 2024-04-12T21:50:19.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:19.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:19.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:19.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 ceph-mon[67226]: mgrmap e35: smithi123.psjrfs(active, since 55s), standbys: smithi177.whybza 2024-04-12T21:50:19.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 bash[67207]: ef4cffbb2b460eb2e393f735c33f4dc979238120664d0918f60d1d7ef52bb3ad 2024-04-12T21:50:20.232 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:19 smithi123 systemd[1]: Started Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:50:20.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:50:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: from='mgr.14768 172.21.15.123:0/609287620' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:50:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: pgmap v30: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 751 op/s 2024-04-12T21:50:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/2001273971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:50:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:50:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: from='client.? 172.21.15.123:0/3776024732' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:50:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: pgmap v31: 97 pgs: 97 active+clean; 326 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 474 op/s 2024-04-12T21:50:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: mon.smithi123 calling monitor election 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: mon.smithi123 is new leader, mons smithi123,smithi177 in quorum (ranks 0,1) 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: monmap epoch 2 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: last_changed 2024-04-12T21:39:39.773252+0000 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: created 2024-04-12T21:37:26.292641+0000 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: min_mon_release 17 (quincy) 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: election_strategy: 1 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.smithi177 2024-04-12T21:50:20.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:50:20.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T21:50:20.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: mgrmap e34: smithi123.psjrfs(active, since 55s), standbys: smithi177.whybza 2024-04-12T21:50:20.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:20.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:20.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:20.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:19 smithi177 ceph-mon[26903]: mgrmap e35: smithi123.psjrfs(active, since 55s), standbys: smithi177.whybza 2024-04-12T21:50:20.516 INFO:teuthology.orchestra.run.smithi123.stdout: 5 156697 11.70 MB/sec execute 180 sec latency 47.081 ms 2024-04-12T21:50:21.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 157110 11.68 MB/sec execute 181 sec latency 35.541 ms 2024-04-12T21:50:22.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:21 smithi177 ceph-mon[26903]: mgrmap e36: smithi123.psjrfs(active, since 56s), standbys: smithi177.whybza 2024-04-12T21:50:22.038 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:21 smithi123 ceph-mon[67226]: mgrmap e36: smithi123.psjrfs(active, since 56s), standbys: smithi177.whybza 2024-04-12T21:50:22.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 157510 11.66 MB/sec execute 182 sec latency 40.172 ms 2024-04-12T21:50:23.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 157949 11.66 MB/sec execute 183 sec latency 39.573 ms 2024-04-12T21:50:24.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 158401 11.64 MB/sec execute 184 sec latency 62.683 ms 2024-04-12T21:50:25.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 158769 11.63 MB/sec execute 185 sec latency 38.472 ms 2024-04-12T21:50:26.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 159127 11.65 MB/sec execute 186 sec latency 41.956 ms 2024-04-12T21:50:27.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 159592 11.68 MB/sec execute 187 sec latency 37.550 ms 2024-04-12T21:50:28.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:28 smithi123 ceph-mon[67226]: from='mgr.? 172.21.15.177:0/2296223663' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:50:28.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:28 smithi123 ceph-mon[67226]: from='mgr.? 172.21.15.177:0/2296223663' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:50:28.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:28 smithi123 ceph-mon[67226]: from='mgr.? 172.21.15.177:0/2296223663' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:50:28.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:28 smithi123 ceph-mon[67226]: from='mgr.? 172.21.15.177:0/2296223663' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:50:28.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:28 smithi123 ceph-mon[67226]: Standby manager daemon smithi177.whybza restarted 2024-04-12T21:50:28.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:28 smithi123 ceph-mon[67226]: Standby manager daemon smithi177.whybza started 2024-04-12T21:50:28.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 159992 11.70 MB/sec execute 188 sec latency 46.843 ms 2024-04-12T21:50:28.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:28 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/2296223663' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/crt"}]: dispatch 2024-04-12T21:50:28.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:28 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/2296223663' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T21:50:28.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:28 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/2296223663' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi177.whybza/key"}]: dispatch 2024-04-12T21:50:28.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:28 smithi177 ceph-mon[26903]: from='mgr.? 172.21.15.177:0/2296223663' entity='mgr.smithi177.whybza' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T21:50:28.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:28 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza restarted 2024-04-12T21:50:28.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:28 smithi177 ceph-mon[26903]: Standby manager daemon smithi177.whybza started 2024-04-12T21:50:29.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:29 smithi123 ceph-mon[67226]: mgrmap e37: smithi123.psjrfs(active, since 64s), standbys: smithi177.whybza 2024-04-12T21:50:29.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 160412 11.66 MB/sec execute 189 sec latency 27.683 ms 2024-04-12T21:50:29.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:29 smithi177 ceph-mon[26903]: mgrmap e37: smithi123.psjrfs(active, since 64s), standbys: smithi177.whybza 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: Active manager daemon smithi123.psjrfs restarted 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: Activating manager daemon smithi123.psjrfs 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: mgrmap e38: smithi123.psjrfs(active, starting, since 0.0218794s), standbys: smithi177.whybza 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi177.whybza", "id": "smithi177.whybza"}]: dispatch 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:50:30.457 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:30.458 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:50:30.459 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:50:30.459 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:50:30.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 160818 11.63 MB/sec execute 190 sec latency 46.296 ms 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: Active manager daemon smithi123.psjrfs restarted 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: Activating manager daemon smithi123.psjrfs 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: mgrmap e38: smithi123.psjrfs(active, starting, since 0.0218794s), standbys: smithi177.whybza 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi123.psjrfs", "id": "smithi123.psjrfs"}]: dispatch 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr metadata", "who": "smithi177.whybza", "id": "smithi177.whybza"}]: dispatch 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:50:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: Manager daemon smithi123.psjrfs is now available 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/mirror_snapshot_schedule"}]: dispatch 2024-04-12T21:50:30.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:30 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.psjrfs/trash_purge_schedule"}]: dispatch 2024-04-12T21:50:31.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 161282 11.63 MB/sec execute 191 sec latency 38.435 ms 2024-04-12T21:50:32.124 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:31 smithi123 ceph-mon[67226]: mgrmap e39: smithi123.psjrfs(active, since 1.03165s), standbys: smithi177.whybza 2024-04-12T21:50:32.125 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:31 smithi123 ceph-mon[67226]: [12/Apr/2024:21:50:31] ENGINE Bus STARTING 2024-04-12T21:50:32.125 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:31 smithi123 ceph-mon[67226]: [12/Apr/2024:21:50:31] ENGINE Serving on https://172.21.15.123:7150 2024-04-12T21:50:32.125 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:31 smithi123 ceph-mon[67226]: [12/Apr/2024:21:50:31] ENGINE Serving on http://172.21.15.123:8765 2024-04-12T21:50:32.125 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:31 smithi123 ceph-mon[67226]: [12/Apr/2024:21:50:31] ENGINE Bus STARTED 2024-04-12T21:50:32.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:31 smithi177 ceph-mon[26903]: mgrmap e39: smithi123.psjrfs(active, since 1.03165s), standbys: smithi177.whybza 2024-04-12T21:50:32.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:31 smithi177 ceph-mon[26903]: [12/Apr/2024:21:50:31] ENGINE Bus STARTING 2024-04-12T21:50:32.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:31 smithi177 ceph-mon[26903]: [12/Apr/2024:21:50:31] ENGINE Serving on https://172.21.15.123:7150 2024-04-12T21:50:32.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:31 smithi177 ceph-mon[26903]: [12/Apr/2024:21:50:31] ENGINE Serving on http://172.21.15.123:8765 2024-04-12T21:50:32.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:31 smithi177 ceph-mon[26903]: [12/Apr/2024:21:50:31] ENGINE Bus STARTED 2024-04-12T21:50:32.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 161819 11.64 MB/sec execute 192 sec latency 203.357 ms 2024-04-12T21:50:33.146 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:32 smithi123 ceph-mon[67226]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T21:50:33.147 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:32 smithi123 ceph-mon[67226]: Cluster is now healthy 2024-04-12T21:50:33.147 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:32 smithi123 ceph-mon[67226]: pgmap v4: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:50:33.147 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:32 smithi123 ceph-mon[67226]: mgrmap e40: smithi123.psjrfs(active, since 2s), standbys: smithi177.whybza 2024-04-12T21:50:33.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:32 smithi177 ceph-mon[26903]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T21:50:33.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:32 smithi177 ceph-mon[26903]: Cluster is now healthy 2024-04-12T21:50:33.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:32 smithi177 ceph-mon[26903]: pgmap v4: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:50:33.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:32 smithi177 ceph-mon[26903]: mgrmap e40: smithi123.psjrfs(active, since 2s), standbys: smithi177.whybza 2024-04-12T21:50:33.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 162256 11.63 MB/sec execute 193 sec latency 39.332 ms 2024-04-12T21:50:34.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 162795 11.69 MB/sec execute 194 sec latency 35.425 ms 2024-04-12T21:50:35.180 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:34 smithi123 ceph-mon[67226]: pgmap v5: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:50:35.181 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:34 smithi123 ceph-mon[67226]: mgrmap e41: smithi123.psjrfs(active, since 4s), standbys: smithi177.whybza 2024-04-12T21:50:35.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:34 smithi177 ceph-mon[26903]: pgmap v5: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:50:35.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:34 smithi177 ceph-mon[26903]: mgrmap e41: smithi123.psjrfs(active, since 4s), standbys: smithi177.whybza 2024-04-12T21:50:35.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 163202 11.71 MB/sec execute 195 sec latency 44.611 ms 2024-04-12T21:50:36.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:36.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:36.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:36 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:36.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:36 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:36.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 163511 11.70 MB/sec execute 196 sec latency 42.028 ms 2024-04-12T21:50:37.477 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:37 smithi123 ceph-mon[67226]: pgmap v6: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:50:37.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:37.478 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:37.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 163737 11.66 MB/sec execute 197 sec latency 46.048 ms 2024-04-12T21:50:37.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:37 smithi177 ceph-mon[26903]: pgmap v6: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-12T21:50:37.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:37 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:37.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:37 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:38.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 163983 11.61 MB/sec execute 198 sec latency 71.166 ms 2024-04-12T21:50:39.275 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:39 smithi123 ceph-mon[67226]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.9 MiB/s wr, 477 op/s 2024-04-12T21:50:39.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 164231 11.58 MB/sec execute 199 sec latency 68.284 ms 2024-04-12T21:50:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:39 smithi177 ceph-mon[26903]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.9 MiB/s wr, 477 op/s 2024-04-12T21:50:40.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 164550 11.56 MB/sec execute 200 sec latency 59.014 ms 2024-04-12T21:50:41.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 165135 11.57 MB/sec execute 201 sec latency 33.866 ms 2024-04-12T21:50:41.819 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:41 smithi123 ceph-mon[67226]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.8 MiB/s wr, 371 op/s 2024-04-12T21:50:41.819 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:41.819 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:41.819 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:41.819 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:42.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:41 smithi177 ceph-mon[26903]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.8 MiB/s wr, 371 op/s 2024-04-12T21:50:42.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:41 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:42.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:41 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:42.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:41 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:42.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:41 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:42.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 165757 11.57 MB/sec execute 202 sec latency 29.656 ms 2024-04-12T21:50:43.517 INFO:teuthology.orchestra.run.smithi123.stdout: 5 166455 11.67 MB/sec execute 203 sec latency 22.895 ms 2024-04-12T21:50:43.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:43 smithi123 ceph-mon[67226]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.0 MiB/s wr, 519 op/s 2024-04-12T21:50:43.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:43.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:43.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:50:43.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:50:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:43 smithi177 ceph-mon[26903]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.0 MiB/s wr, 519 op/s 2024-04-12T21:50:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:43 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:43 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:43 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:50:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:43 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:50:44.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 167089 11.71 MB/sec execute 204 sec latency 42.167 ms 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: Updating smithi177:/etc/ceph/ceph.conf 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:44.834 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:44.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:44.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:44.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:50:44.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.conf 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: Updating smithi177:/etc/ceph/ceph.conf 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: Updating smithi177:/etc/ceph/ceph.client.admin.keyring 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:50:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:44 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T21:50:45.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 167596 11.70 MB/sec execute 205 sec latency 21.864 ms 2024-04-12T21:50:45.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:45 smithi123 ceph-mon[67226]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 4.6 MiB/s wr, 474 op/s 2024-04-12T21:50:45.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:45 smithi123 ceph-mon[67226]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:50:45.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:45 smithi123 ceph-mon[67226]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:50:45.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:45 smithi123 ceph-mon[67226]: pgmap v11: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-04-12T21:50:45.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:45 smithi123 ceph-mon[67226]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T21:50:45.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:50:46.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:45 smithi177 ceph-mon[26903]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 4.6 MiB/s wr, 474 op/s 2024-04-12T21:50:46.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:45 smithi177 ceph-mon[26903]: Updating smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:50:46.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:45 smithi177 ceph-mon[26903]: Updating smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T21:50:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:45 smithi177 ceph-mon[26903]: pgmap v11: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-04-12T21:50:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:45 smithi177 ceph-mon[26903]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T21:50:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:45 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:50:46.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 168109 11.68 MB/sec execute 206 sec latency 34.844 ms 2024-04-12T21:50:47.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 168398 11.66 MB/sec execute 207 sec latency 55.022 ms 2024-04-12T21:50:47.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:47 smithi123 ceph-mon[67226]: pgmap v12: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 9.4 MiB/s wr, 946 op/s 2024-04-12T21:50:47.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:47 smithi123 ceph-mon[67226]: Upgrade: Updating mon.smithi177 2024-04-12T21:50:47.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:47 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:47.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:47 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:50:47.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:47 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:50:47.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:47 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:50:47.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:47 smithi123 ceph-mon[67226]: Deploying daemon mon.smithi177 on smithi177 2024-04-12T21:50:48.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:47 smithi177 ceph-mon[26903]: pgmap v12: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 9.4 MiB/s wr, 946 op/s 2024-04-12T21:50:48.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:47 smithi177 ceph-mon[26903]: Upgrade: Updating mon.smithi177 2024-04-12T21:50:48.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:47 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:48.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:47 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:50:48.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:47 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:50:48.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:47 smithi177 ceph-mon[26903]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:50:48.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:47 smithi177 ceph-mon[26903]: Deploying daemon mon.smithi177 on smithi177 2024-04-12T21:50:48.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 168819 11.66 MB/sec execute 208 sec latency 51.236 ms 2024-04-12T21:50:49.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:48 smithi177 systemd[1]: Stopping Ceph mon.smithi177 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:50:49.170 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:50:49.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 169193 11.63 MB/sec execute 209 sec latency 40.626 ms 2024-04-12T21:50:49.518 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:49 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177[26873]: 2024-04-12T21:50:49.140+0000 7f93c9650700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi177 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:50:49.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:49 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177[26873]: 2024-04-12T21:50:49.140+0000 7f93c9650700 -1 mon.smithi177@1(peon) e2 *** Got Signal Terminated *** 2024-04-12T21:50:49.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:49 smithi177 podman[52710]: 2024-04-12 21:50:49.21251673 +0000 UTC m=+0.195049056 container died ae9bbb3ebc295646936f21f06a8ebbc63123b4ec3785a080e57ac456af785a37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, name=centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , version=8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public) 2024-04-12T21:50:49.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:49 smithi177 podman[52710]: 2024-04-12 21:50:49.617573797 +0000 UTC m=+0.600106116 container cleanup ae9bbb3ebc295646936f21f06a8ebbc63123b4ec3785a080e57ac456af785a37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, RELEASE=HEAD, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, version=8, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , release=754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-12T21:50:49.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:49 smithi177 bash[52710]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177 2024-04-12T21:50:50.077 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:50:50.077 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (2m) 8s ago 11m 25.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:50:50.077 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (11m) 8s ago 11m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:50:50.077 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (11m) 15s ago 11m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:50:50.077 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 unknown 8s ago 11m - - 2024-04-12T21:50:50.077 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (6m) 8s ago 6m 38.5M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (6m) 15s ago 6m 14.1M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (104s) 8s ago 13m 479M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (50s) 15s ago 11m 447M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (31s) 8s ago 13m 30.6M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (11m) 15s ago 11m 52.6M 2048M 17.2.0 e1d6a67b021e ae9bbb3ebc29 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (5m) 8s ago 5m 256M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (5m) 15s ago 5m 84.3M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (2m) 8s ago 11m 21.7M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (113s) 15s ago 11m 16.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (10m) 8s ago 10m 573M 1936M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (10m) 8s ago 10m 482M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (9m) 8s ago 9m 349M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (9m) 8s ago 9m 493M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:50:50.078 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (8m) 15s ago 8m 524M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:50:50.079 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (8m) 15s ago 8m 329M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:50:50.079 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (7m) 15s ago 7m 522M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:50:50.079 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (7m) 15s ago 7m 436M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:50:50.079 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (57s) 8s ago 10m 56.1M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:50:50.259 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:49 smithi177 podman[52724]: 2024-04-12 21:50:49.877908892 +0000 UTC m=+0.661659828 container remove ae9bbb3ebc295646936f21f06a8ebbc63123b4ec3785a080e57ac456af785a37 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, name=centos-stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-type=git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-12T21:50:50.259 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi177.service: Deactivated successfully. 2024-04-12T21:50:50.259 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 systemd[1]: Stopped Ceph mon.smithi177 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:50:50.259 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi177.service: Consumed 7.150s CPU time. 2024-04-12T21:50:50.259 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 systemd[1]: Starting Ceph mon.smithi177 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:50:50.513 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 podman[52851]: 2024-04-12 21:50:50.385785972 +0000 UTC m=+0.020856287 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:50:50.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 169498 11.64 MB/sec execute 210 sec latency 35.358 ms 2024-04-12T21:50:50.655 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:50:50.655 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:50:50.655 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T21:50:50.655 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:50:50.655 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:50:50.655 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 3 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:50:50.656 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:50:50.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 podman[52851]: 2024-04-12 21:50:50.512368602 +0000 UTC m=+0.147438914 container create c090c7cd3f2e49bc65ed5296331b4c776d1022deb77bb5f02fc1352aaade08f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-12T21:50:50.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 podman[52851]: 2024-04-12 21:50:50.757131106 +0000 UTC m=+0.392201420 container init c090c7cd3f2e49bc65ed5296331b4c776d1022deb77bb5f02fc1352aaade08f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-12T21:50:50.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 podman[52851]: 2024-04-12 21:50:50.760319017 +0000 UTC m=+0.395389328 container start c090c7cd3f2e49bc65ed5296331b4c776d1022deb77bb5f02fc1352aaade08f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-12T21:50:51.095 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev), process ceph-mon, pid 2 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: pidfile_write: ignore empty --pid-file 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: load: jerasure load: lrc 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: RocksDB version: 7.9.2 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Git sha 0 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Compile date 2024-04-09 21:18:54 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: DB SUMMARY 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: DB Session ID: TO9MRCXSV1IY333YIIGC 2024-04-12T21:50:51.096 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: CURRENT file: CURRENT 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi177/store.db dir, Total Num: 1, files: 000024.sst 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi177/store.db: 000022.log size: 6929826 ; 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.error_if_exists: 0 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.create_if_missing: 0 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.paranoid_checks: 1 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T21:50:51.097 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.env: 0x55803b072c60 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.info_log: 0x55803c3799a0 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.statistics: (nil) 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.use_fsync: 0 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_log_file_size: 0 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T21:50:51.098 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.allow_fallocate: 1 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.use_direct_reads: 0 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.db_log_dir: 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.wal_dir: 2024-04-12T21:50:51.099 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T21:50:51.100 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T21:50:51.100 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.write_buffer_manager: 0x55803c34c1e0 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T21:50:51.101 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.unordered_write: 0 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.row_cache: None 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.wal_filter: None 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.two_write_queues: 0 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.wal_compression: 0 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.atomic_flush: 0 2024-04-12T21:50:51.102 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.log_readahead_size: 0 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_background_jobs: 2 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_background_compactions: -1 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_subcompactions: 1 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T21:50:51.103 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_open_files: -1 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_background_flushes: -1 2024-04-12T21:50:51.104 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Compression algorithms supported: 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: kZSTD supported: 0 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: kXpressCompression supported: 0 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: kZlibCompression supported: 1 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: kSnappyCompression supported: 1 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: kLZ4Compression supported: 1 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: kBZip2Compression supported: 0 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi177/store.db/MANIFEST-000009 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T21:50:51.105 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.merge_operator: 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_filter: None 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55803c379aa0) 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: cache_index_and_filter_blocks: 1 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: pin_top_level_index_and_filter: 1 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: index_type: 0 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: data_block_index_type: 0 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: index_shortening: 1 2024-04-12T21:50:51.106 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: checksum: 4 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: no_block_cache: 0 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_cache: 0x55803c37d090 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_cache_name: BinnedLRUCache 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_cache_options: 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: capacity : 536870912 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: num_shard_bits : 4 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: strict_capacity_limit : 0 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: high_pri_pool_ratio: 0.000 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_cache_compressed: (nil) 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: persistent_cache: (nil) 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_size: 4096 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_size_deviation: 10 2024-04-12T21:50:51.107 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_restart_interval: 16 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: index_block_restart_interval: 1 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: metadata_block_size: 4096 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: partition_filters: 0 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: use_delta_encoding: 1 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: filter_policy: bloomfilter 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: whole_key_filtering: 1 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: verify_compression: 0 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: read_amp_bytes_per_bit: 0 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: format_version: 5 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: enable_index_compression: 1 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: block_align: 0 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: max_auto_readahead_size: 262144 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: prepopulate_block_cache: 0 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: initial_auto_readahead_size: 8192 2024-04-12T21:50:51.108 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression: NoCompression 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.num_levels: 7 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T21:50:51.109 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T21:50:51.110 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T21:50:51.111 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T21:50:51.112 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.table_properties_collectors: 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.inplace_update_support: 0 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.bloom_locality: 0 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.max_successive_merges: 0 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T21:50:51.113 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.ttl: 2592000 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.enable_blob_files: false 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.min_blob_size: 0 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T21:50:51.114 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi177/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12245, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 93fb56e4-d4a5-4bee-b106-6b8efbf3c9b9 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712958650790563, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712958650807524, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4228566, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12239, "largest_seqno": 13952, "table_properties": {"data_size": 4220934, "index_size": 4885, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15995, "raw_average_key_size": 23, "raw_value_size": 4205966, "raw_average_value_size": 6122, "num_data_blocks": 222, "num_entries": 687, "num_filter_entries": 687, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712958650, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "93fb56e4-d4a5-4bee-b106-6b8efbf3c9b9", "db_session_id": "TO9MRCXSV1IY333YIIGC", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712958650807665, "job": 1, "event": "recovery_finished"} 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi177/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55803c46e000 2024-04-12T21:50:51.115 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: DB pointer 0x55803c458000 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** DB Stats ** 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** Compaction Stats [default] ** 2024-04-12T21:50:51.116 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: L0 1/0 4.03 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 515.8 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: L6 1/0 6.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Sum 2/0 10.80 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 515.8 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 515.8 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** Compaction Stats [default] ** 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 515.8 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T21:50:51.117 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Cumulative compaction: 0.00 GB write, 202.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Interval compaction: 0.00 GB write, 202.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Block cache BinnedLRUCache@0x55803c37d090#2 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T21:50:51.118 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: starting mon.smithi177 rank 1 at public addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] at bind addrs [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi177 fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: mon.smithi177@-1(???) e2 preinit fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: mon.smithi177@-1(???).mds e6 new map 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: mon.smithi177@-1(???).mds e6 print_map 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: e6 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: legacy client fscid: 1 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Filesystem 'foofs' (1) 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: fs_name foofs 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: epoch 5 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: created 2024-04-12T21:44:41.570840+0000 2024-04-12T21:50:51.119 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: modified 2024-04-12T21:44:47.590655+0000 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: tableserver 0 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: root 0 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: session_timeout 60 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: session_autoclose 300 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: max_file_size 1099511627776 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: max_xattr_size 65536 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: required_client_features {} 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: last_failure 0 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: last_failure_osd_epoch 0 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: max_mds 1 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: in 0 2024-04-12T21:50:51.120 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: up {0=14486} 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: failed 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: damaged 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: stopped 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: data_pools [3] 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: metadata_pool 2 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: inline_data disabled 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: balancer 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: bal_rank_mask -1 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: standby_count_wanted 1 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: qdb_cluster leader: 0 members: 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: [mds.foofs.smithi123.poxexz{0:14486} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.123:6834/1104424315,v1:172.21.15.123:6835/1104424315] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: Standby daemons: 2024-04-12T21:50:51.121 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout: [mds.foofs.smithi177.hqctzz{-1:24269} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: mon.smithi177@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: mon.smithi177@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: mon.smithi177@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: mon.smithi177@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 ceph-mon[52869]: mon.smithi177@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 bash[52851]: c090c7cd3f2e49bc65ed5296331b4c776d1022deb77bb5f02fc1352aaade08f0 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:50 smithi177 systemd[1]: Started Ceph mon.smithi177 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:50:51.122 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:51 smithi177 ceph-mon[52869]: pgmap v13: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 634 op/s 2024-04-12T21:50:51.184 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:50:51.184 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:50:51.184 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:50:51.185 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:50:51.185 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:50:51.185 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr" 2024-04-12T21:50:51.185 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:50:51.185 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "3/23 daemons upgraded", 2024-04-12T21:50:51.185 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mon daemons", 2024-04-12T21:50:51.185 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:50:51.185 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:50:51.185 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:51 smithi123 ceph-mon[67226]: pgmap v13: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 634 op/s 2024-04-12T21:50:51.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 169783 11.63 MB/sec execute 211 sec latency 54.487 ms 2024-04-12T21:50:51.798 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:51.798 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:51.798 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:52.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: election_strategy: 1 2024-04-12T21:50:52.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-04-12T21:50:52.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.smithi177 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: mgrmap e41: smithi123.psjrfs(active, since 21s), standbys: smithi177.whybza 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: mon.smithi177 calling monitor election 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: mon.smithi123 calling monitor election 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: mon.smithi123 is new leader, mons smithi123,smithi177 in quorum (ranks 0,1) 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:50:52.195 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: monmap epoch 3 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: last_changed 2024-04-12T21:50:51.097807+0000 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: created 2024-04-12T21:37:26.292641+0000 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: min_mon_release 19 (squid) 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: election_strategy: 1 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.smithi177 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: mgrmap e41: smithi123.psjrfs(active, since 21s), standbys: smithi177.whybza 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:52.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:52.197 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:52.197 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:50:52.197 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:52 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3747873204' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:50:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: election_strategy: 1 2024-04-12T21:50:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-04-12T21:50:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.smithi177 2024-04-12T21:50:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:50:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T21:50:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: mgrmap e41: smithi123.psjrfs(active, since 21s), standbys: smithi177.whybza 2024-04-12T21:50:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: mon.smithi177 calling monitor election 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: mon.smithi123 calling monitor election 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: mon.smithi123 is new leader, mons smithi123,smithi177 in quorum (ranks 0,1) 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mon metadata", "id": "smithi177"}]: dispatch 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: monmap epoch 3 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: fsid c8b0510a-f914-11ee-bc8d-c7b262605968 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: last_changed 2024-04-12T21:50:51.097807+0000 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: created 2024-04-12T21:37:26.292641+0000 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: min_mon_release 19 (squid) 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: election_strategy: 1 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-04-12T21:50:52.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: 1: [v2:172.21.15.177:3300/0,v1:172.21.15.177:6789/0] mon.smithi177 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: mgrmap e41: smithi123.psjrfs(active, since 21s), standbys: smithi177.whybza 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:50:52.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:52 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3747873204' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:50:52.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 170182 11.65 MB/sec execute 212 sec latency 38.940 ms 2024-04-12T21:50:53.224 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:53 smithi123 ceph-mon[67226]: pgmap v15: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.5 MiB/s wr, 657 op/s 2024-04-12T21:50:53.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 170539 11.66 MB/sec execute 213 sec latency 51.383 ms 2024-04-12T21:50:53.521 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:53 smithi177 ceph-mon[52869]: pgmap v15: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.5 MiB/s wr, 657 op/s 2024-04-12T21:50:54.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 170750 11.62 MB/sec execute 214 sec latency 47.237 ms 2024-04-12T21:50:55.519 INFO:teuthology.orchestra.run.smithi123.stdout: 5 170942 11.58 MB/sec execute 215 sec latency 53.647 ms 2024-04-12T21:50:55.811 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:55 smithi123 ceph-mon[67226]: pgmap v16: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.5 MiB/s wr, 657 op/s 2024-04-12T21:50:55.812 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:55.812 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:56.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:55 smithi177 ceph-mon[52869]: pgmap v16: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.5 MiB/s wr, 657 op/s 2024-04-12T21:50:56.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:56.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:56.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 171178 11.55 MB/sec execute 216 sec latency 63.028 ms 2024-04-12T21:50:57.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 171465 11.51 MB/sec execute 217 sec latency 65.867 ms 2024-04-12T21:50:57.637 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:57 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:57.638 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:57 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:57.638 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:57 smithi123 ceph-mon[67226]: pgmap v17: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 757 op/s 2024-04-12T21:50:57.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:57 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:57.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:57 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:50:57.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:57 smithi177 ceph-mon[52869]: pgmap v17: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 757 op/s 2024-04-12T21:50:58.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 171736 11.49 MB/sec execute 218 sec latency 55.794 ms 2024-04-12T21:50:59.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 172100 11.48 MB/sec execute 219 sec latency 50.739 ms 2024-04-12T21:50:59.687 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:50:59 smithi123 ceph-mon[67226]: pgmap v18: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-04-12T21:51:00.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:50:59 smithi177 ceph-mon[52869]: pgmap v18: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-04-12T21:51:00.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 172443 11.47 MB/sec execute 220 sec latency 61.425 ms 2024-04-12T21:51:00.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:51:00.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:00.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:00.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:00.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:51:00.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:51:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:51:00.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:01.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 172806 11.44 MB/sec execute 221 sec latency 54.574 ms 2024-04-12T21:51:01.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-04-12T21:51:01.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 469 op/s 2024-04-12T21:51:01.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.1 MiB/s wr, 262 op/s 2024-04-12T21:51:01.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:01.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: Reconfiguring mon.smithi123 (monmap changed)... 2024-04-12T21:51:01.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:51:01.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:51:01.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:01.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:01 smithi123 ceph-mon[67226]: Reconfiguring daemon mon.smithi123 on smithi123 2024-04-12T21:51:02.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-04-12T21:51:02.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 469 op/s 2024-04-12T21:51:02.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.1 MiB/s wr, 262 op/s 2024-04-12T21:51:02.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:02.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: Reconfiguring mon.smithi123 (monmap changed)... 2024-04-12T21:51:02.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:51:02.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:51:02.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:02.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:01 smithi177 ceph-mon[52869]: Reconfiguring daemon mon.smithi123 on smithi123 2024-04-12T21:51:02.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 173069 11.44 MB/sec execute 222 sec latency 52.901 ms 2024-04-12T21:51:03.519 INFO:teuthology.orchestra.run.smithi123.stdout: 5 173334 11.43 MB/sec execute 223 sec latency 51.834 ms 2024-04-12T21:51:03.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:03 smithi123 ceph-mon[67226]: pgmap v22: 97 pgs: 97 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.3 MiB/s wr, 553 op/s 2024-04-12T21:51:03.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:03.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:03.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.psjrfs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:51:03.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:51:03.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:03 smithi177 ceph-mon[52869]: pgmap v22: 97 pgs: 97 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.3 MiB/s wr, 553 op/s 2024-04-12T21:51:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.psjrfs", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:51:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:51:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:04.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 173644 11.44 MB/sec execute 224 sec latency 56.651 ms 2024-04-12T21:51:04.787 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:04 smithi123 ceph-mon[67226]: Reconfiguring mgr.smithi123.psjrfs (monmap changed)... 2024-04-12T21:51:04.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:04 smithi123 ceph-mon[67226]: Reconfiguring daemon mgr.smithi123.psjrfs on smithi123 2024-04-12T21:51:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:04 smithi177 ceph-mon[52869]: Reconfiguring mgr.smithi123.psjrfs (monmap changed)... 2024-04-12T21:51:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:04 smithi177 ceph-mon[52869]: Reconfiguring daemon mgr.smithi123.psjrfs on smithi123 2024-04-12T21:51:05.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 173966 11.43 MB/sec execute 225 sec latency 55.544 ms 2024-04-12T21:51:05.796 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:05 smithi123 ceph-mon[67226]: pgmap v23: 97 pgs: 97 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.2 MiB/s wr, 291 op/s 2024-04-12T21:51:05.797 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:05.797 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:05.797 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:51:05.797 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:06.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:05 smithi177 ceph-mon[52869]: pgmap v23: 97 pgs: 97 active+clean; 324 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.2 MiB/s wr, 291 op/s 2024-04-12T21:51:06.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:06.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:06.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:51:06.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:06.519 INFO:teuthology.orchestra.run.smithi123.stdout: 5 174270 11.43 MB/sec execute 226 sec latency 42.243 ms 2024-04-12T21:51:06.818 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:06 smithi123 ceph-mon[67226]: Reconfiguring crash.smithi123 (monmap changed)... 2024-04-12T21:51:06.819 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:06 smithi123 ceph-mon[67226]: Reconfiguring daemon crash.smithi123 on smithi123 2024-04-12T21:51:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:06 smithi177 ceph-mon[52869]: Reconfiguring crash.smithi123 (monmap changed)... 2024-04-12T21:51:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:06 smithi177 ceph-mon[52869]: Reconfiguring daemon crash.smithi123 on smithi123 2024-04-12T21:51:07.518 INFO:teuthology.orchestra.run.smithi123.stdout: 5 174576 11.40 MB/sec execute 227 sec latency 32.199 ms 2024-04-12T21:51:07.837 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:07 smithi123 ceph-mon[67226]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 554 op/s 2024-04-12T21:51:08.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:07 smithi177 ceph-mon[52869]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 554 op/s 2024-04-12T21:51:08.519 INFO:teuthology.orchestra.run.smithi123.stdout: 5 174895 11.37 MB/sec execute 228 sec latency 55.451 ms 2024-04-12T21:51:09.520 INFO:teuthology.orchestra.run.smithi123.stdout: 5 175177 11.34 MB/sec execute 229 sec latency 57.418 ms 2024-04-12T21:51:09.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:09.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:09.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:09 smithi123 ceph-mon[67226]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T21:51:09.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T21:51:09.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:09.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:09 smithi123 ceph-mon[67226]: Reconfiguring daemon osd.0 on smithi123 2024-04-12T21:51:09.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:09 smithi123 ceph-mon[67226]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 553 op/s 2024-04-12T21:51:09.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:09.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:09.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:09 smithi177 ceph-mon[52869]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T21:51:09.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T21:51:09.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:09.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:09 smithi177 ceph-mon[52869]: Reconfiguring daemon osd.0 on smithi123 2024-04-12T21:51:09.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:09 smithi177 ceph-mon[52869]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 553 op/s 2024-04-12T21:51:10.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 175443 11.31 MB/sec execute 230 sec latency 50.124 ms 2024-04-12T21:51:11.520 INFO:teuthology.orchestra.run.smithi123.stdout: 5 175754 11.32 MB/sec execute 231 sec latency 82.037 ms 2024-04-12T21:51:11.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:11 smithi123 ceph-mon[67226]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 446 op/s 2024-04-12T21:51:11.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:11 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:11.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:11 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:11.921 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:11 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T21:51:11.921 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:11 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:12.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:11 smithi177 ceph-mon[52869]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 446 op/s 2024-04-12T21:51:12.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:11 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:12.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:11 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:12.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:11 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T21:51:12.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:11 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:12.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 176061 11.29 MB/sec execute 232 sec latency 89.539 ms 2024-04-12T21:51:12.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:12 smithi123 ceph-mon[67226]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T21:51:12.947 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:12 smithi123 ceph-mon[67226]: Reconfiguring daemon osd.1 on smithi123 2024-04-12T21:51:13.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:12 smithi177 ceph-mon[52869]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T21:51:13.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:12 smithi177 ceph-mon[52869]: Reconfiguring daemon osd.1 on smithi123 2024-04-12T21:51:13.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 176497 11.30 MB/sec execute 233 sec latency 33.661 ms 2024-04-12T21:51:13.722 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:13 smithi123 ceph-mon[67226]: pgmap v27: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.4 MiB/s wr, 570 op/s 2024-04-12T21:51:14.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:13 smithi177 ceph-mon[52869]: pgmap v27: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.4 MiB/s wr, 570 op/s 2024-04-12T21:51:14.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 176736 11.28 MB/sec execute 234 sec latency 48.120 ms 2024-04-12T21:51:15.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 177097 11.30 MB/sec execute 235 sec latency 57.520 ms 2024-04-12T21:51:15.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:15 smithi123 ceph-mon[67226]: pgmap v28: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 374 op/s 2024-04-12T21:51:15.791 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:51:16.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:15 smithi177 ceph-mon[52869]: pgmap v28: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 374 op/s 2024-04-12T21:51:16.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:51:16.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 177477 11.30 MB/sec execute 236 sec latency 42.489 ms 2024-04-12T21:51:16.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:16.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:16.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:16 smithi123 ceph-mon[67226]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T21:51:16.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T21:51:16.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:16.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:16 smithi123 ceph-mon[67226]: Reconfiguring daemon osd.2 on smithi123 2024-04-12T21:51:17.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:17.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:17.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:16 smithi177 ceph-mon[52869]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T21:51:17.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T21:51:17.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:17.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:16 smithi177 ceph-mon[52869]: Reconfiguring daemon osd.2 on smithi123 2024-04-12T21:51:17.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 177868 11.31 MB/sec execute 237 sec latency 42.136 ms 2024-04-12T21:51:17.824 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:17 smithi123 ceph-mon[67226]: pgmap v29: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-04-12T21:51:18.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:17 smithi177 ceph-mon[52869]: pgmap v29: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 565 op/s 2024-04-12T21:51:18.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 178233 11.28 MB/sec execute 238 sec latency 25.135 ms 2024-04-12T21:51:19.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 178658 11.26 MB/sec execute 239 sec latency 39.316 ms 2024-04-12T21:51:19.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:19.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:19.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:19 smithi123 ceph-mon[67226]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T21:51:19.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T21:51:19.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:19.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:19 smithi123 ceph-mon[67226]: Reconfiguring daemon osd.3 on smithi123 2024-04-12T21:51:19.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:19 smithi123 ceph-mon[67226]: pgmap v30: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-04-12T21:51:19.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:19.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:19.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:19 smithi177 ceph-mon[52869]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T21:51:19.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T21:51:19.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:19.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:19 smithi177 ceph-mon[52869]: Reconfiguring daemon osd.3 on smithi123 2024-04-12T21:51:19.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:19 smithi177 ceph-mon[52869]: pgmap v30: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-04-12T21:51:20.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 179027 11.25 MB/sec execute 240 sec latency 32.143 ms 2024-04-12T21:51:21.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 179449 11.26 MB/sec execute 241 sec latency 42.503 ms 2024-04-12T21:51:21.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:21 smithi123 ceph-mon[67226]: pgmap v31: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-04-12T21:51:21.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:21.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:21.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.poxexz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:51:21.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:22.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:21 smithi177 ceph-mon[52869]: pgmap v31: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-04-12T21:51:22.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:22.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:22.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.poxexz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:51:22.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:22.370 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:51:22.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 179971 11.25 MB/sec execute 242 sec latency 46.161 ms 2024-04-12T21:51:22.898 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:22 smithi123 ceph-mon[67226]: Reconfiguring mds.foofs.smithi123.poxexz (monmap changed)... 2024-04-12T21:51:22.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:22 smithi123 ceph-mon[67226]: Reconfiguring daemon mds.foofs.smithi123.poxexz on smithi123 2024-04-12T21:51:23.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:22 smithi177 ceph-mon[52869]: Reconfiguring mds.foofs.smithi123.poxexz (monmap changed)... 2024-04-12T21:51:23.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:22 smithi177 ceph-mon[52869]: Reconfiguring daemon mds.foofs.smithi123.poxexz on smithi123 2024-04-12T21:51:23.431 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:51:23.431 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (3m) 41s ago 12m 25.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:51:23.431 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (12m) 41s ago 12m 6400k - 17.2.0 e1d6a67b021e f0ee7cb501eb 2024-04-12T21:51:23.431 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (11m) 28s ago 11m 6405k - 17.2.0 e1d6a67b021e e7961756bc2b 2024-04-12T21:51:23.431 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 unknown 41s ago 12m - - 2024-04-12T21:51:23.431 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (6m) 41s ago 6m 38.5M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (6m) 28s ago 6m 14.5M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (2m) 41s ago 13m 479M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (83s) 28s ago 11m 448M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (64s) 41s ago 13m 30.6M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (33s) 28s ago 11m 24.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (6m) 41s ago 6m 256M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (6m) 28s ago 6m 84.5M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (2m) 41s ago 12m 21.7M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (2m) 28s ago 11m 18.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (11m) 41s ago 11m 573M 1936M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (10m) 41s ago 10m 482M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (10m) 41s ago 10m 349M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (9m) 41s ago 9m 493M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:51:23.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (9m) 28s ago 9m 538M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:51:23.433 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (8m) 28s ago 8m 372M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:51:23.433 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (8m) 28s ago 8m 556M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:51:23.433 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (8m) 28s ago 8m 473M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:51:23.433 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (90s) 41s ago 11m 56.1M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:51:23.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 180344 11.27 MB/sec execute 243 sec latency 27.733 ms 2024-04-12T21:51:23.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:23 smithi123 ceph-mon[67226]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:23.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:23 smithi123 ceph-mon[67226]: pgmap v32: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 640 op/s 2024-04-12T21:51:23.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:23 smithi123 ceph-mon[67226]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:23.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:23 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:23.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:23 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:23.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:23 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:51:23.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:23 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:23 smithi177 ceph-mon[52869]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:23 smithi177 ceph-mon[52869]: pgmap v32: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 640 op/s 2024-04-12T21:51:24.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:23 smithi177 ceph-mon[52869]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:24.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:23 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:24.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:23 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:24.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:23 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:51:24.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:23 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:51:24.098 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:51:24.099 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:51:24.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 180863 11.30 MB/sec execute 244 sec latency 36.458 ms 2024-04-12T21:51:24.661 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:51:24.661 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:51:24.661 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:51:24.661 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:51:24.662 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:51:24.662 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:51:24.662 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-04-12T21:51:24.662 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:51:24.662 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "4/23 daemons upgraded", 2024-04-12T21:51:24.662 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mon daemons", 2024-04-12T21:51:24.662 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:51:24.663 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:51:24.917 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:24 smithi123 ceph-mon[67226]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:24 smithi123 ceph-mon[67226]: Reconfiguring crash.smithi177 (monmap changed)... 2024-04-12T21:51:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:24 smithi123 ceph-mon[67226]: Reconfiguring daemon crash.smithi177 on smithi177 2024-04-12T21:51:24.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:24 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/599787053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:51:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:24 smithi177 ceph-mon[52869]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:24 smithi177 ceph-mon[52869]: Reconfiguring crash.smithi177 (monmap changed)... 2024-04-12T21:51:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:24 smithi177 ceph-mon[52869]: Reconfiguring daemon crash.smithi177 on smithi177 2024-04-12T21:51:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:24 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/599787053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:51:25.343 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:51:25.343 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:51:25.344 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:51:25.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 181304 11.31 MB/sec execute 245 sec latency 35.686 ms 2024-04-12T21:51:25.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:25 smithi123 ceph-mon[67226]: pgmap v33: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-04-12T21:51:25.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:25 smithi123 ceph-mon[67226]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:25.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:25 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1205621707' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:51:25.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:25.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:25.925 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:51:25.925 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:51:25.925 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:25 smithi177 ceph-mon[52869]: pgmap v33: 97 pgs: 97 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-04-12T21:51:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:25 smithi177 ceph-mon[52869]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:25 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1205621707' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:51:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi177.whybza", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T21:51:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T21:51:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:26.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 181700 11.30 MB/sec execute 246 sec latency 34.742 ms 2024-04-12T21:51:26.932 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:26 smithi123 ceph-mon[67226]: Reconfiguring mgr.smithi177.whybza (monmap changed)... 2024-04-12T21:51:26.933 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:26 smithi123 ceph-mon[67226]: Reconfiguring daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:51:27.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:26 smithi177 ceph-mon[52869]: Reconfiguring mgr.smithi177.whybza (monmap changed)... 2024-04-12T21:51:27.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:26 smithi177 ceph-mon[52869]: Reconfiguring daemon mgr.smithi177.whybza on smithi177 2024-04-12T21:51:27.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 182157 11.28 MB/sec execute 247 sec latency 51.953 ms 2024-04-12T21:51:27.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:27 smithi123 ceph-mon[67226]: pgmap v34: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.5 MiB/s wr, 711 op/s 2024-04-12T21:51:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:27 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:27 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:27 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:51:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:27 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:51:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:27 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:27.968 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:27 smithi177 ceph-mon[52869]: pgmap v34: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.5 MiB/s wr, 711 op/s 2024-04-12T21:51:27.969 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:27 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:27.969 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:27 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:27.969 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:27 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T21:51:27.969 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:27 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T21:51:27.969 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:27 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:28.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 182581 11.28 MB/sec execute 248 sec latency 42.272 ms 2024-04-12T21:51:28.712 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:28 smithi123 ceph-mon[67226]: Reconfiguring mon.smithi177 (monmap changed)... 2024-04-12T21:51:28.713 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:28 smithi123 ceph-mon[67226]: Reconfiguring daemon mon.smithi177 on smithi177 2024-04-12T21:51:28.973 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:28 smithi177 ceph-mon[52869]: Reconfiguring mon.smithi177 (monmap changed)... 2024-04-12T21:51:28.973 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:28 smithi177 ceph-mon[52869]: Reconfiguring daemon mon.smithi177 on smithi177 2024-04-12T21:51:29.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 183405 11.31 MB/sec execute 249 sec latency 19.078 ms 2024-04-12T21:51:29.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:29 smithi123 ceph-mon[67226]: pgmap v35: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 521 op/s 2024-04-12T21:51:29.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:29 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:29.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:29 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:29.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:29 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T21:51:29.720 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:29 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:29.970 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:29 smithi177 ceph-mon[52869]: pgmap v35: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 521 op/s 2024-04-12T21:51:29.970 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:29 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:29.970 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:29 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:29.970 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:29 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T21:51:29.970 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:29 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:30.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 184027 11.35 MB/sec execute 250 sec latency 30.127 ms 2024-04-12T21:51:30.727 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:30 smithi123 ceph-mon[67226]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T21:51:30.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:30 smithi123 ceph-mon[67226]: Reconfiguring daemon osd.4 on smithi177 2024-04-12T21:51:30.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:51:30.974 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:30 smithi177 ceph-mon[52869]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T21:51:30.974 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:30 smithi177 ceph-mon[52869]: Reconfiguring daemon osd.4 on smithi177 2024-04-12T21:51:30.974 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:51:31.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 184851 11.42 MB/sec execute 251 sec latency 29.312 ms 2024-04-12T21:51:31.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:31 smithi123 ceph-mon[67226]: pgmap v36: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 521 op/s 2024-04-12T21:51:31.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:31.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:31.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T21:51:31.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:31.988 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:31 smithi177 ceph-mon[52869]: pgmap v36: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 521 op/s 2024-04-12T21:51:31.988 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:31.988 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:31.988 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T21:51:31.988 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:32.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 185192 11.41 MB/sec execute 252 sec latency 36.132 ms 2024-04-12T21:51:32.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:32 smithi123 ceph-mon[67226]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T21:51:32.752 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:32 smithi123 ceph-mon[67226]: Reconfiguring daemon osd.5 on smithi177 2024-04-12T21:51:33.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:32 smithi177 ceph-mon[52869]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T21:51:33.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:32 smithi177 ceph-mon[52869]: Reconfiguring daemon osd.5 on smithi177 2024-04-12T21:51:33.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 185503 11.38 MB/sec execute 253 sec latency 59.404 ms 2024-04-12T21:51:34.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:33 smithi123 ceph-mon[67226]: pgmap v37: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 867 op/s 2024-04-12T21:51:34.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:34.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:34.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:33 smithi123 ceph-mon[67226]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T21:51:34.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T21:51:34.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:34.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:33 smithi123 ceph-mon[67226]: Reconfiguring daemon osd.6 on smithi177 2024-04-12T21:51:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:33 smithi177 ceph-mon[52869]: pgmap v37: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 867 op/s 2024-04-12T21:51:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:33 smithi177 ceph-mon[52869]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T21:51:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T21:51:34.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:34.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:33 smithi177 ceph-mon[52869]: Reconfiguring daemon osd.6 on smithi177 2024-04-12T21:51:34.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 185796 11.35 MB/sec execute 254 sec latency 58.741 ms 2024-04-12T21:51:35.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 186108 11.33 MB/sec execute 255 sec latency 50.441 ms 2024-04-12T21:51:35.837 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:35 smithi123 ceph-mon[67226]: pgmap v38: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.5 MiB/s wr, 615 op/s 2024-04-12T21:51:35.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:35 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:35.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:35 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:35.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:35 smithi123 ceph-mon[67226]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T21:51:35.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:35 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T21:51:35.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:35 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:35.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:35 smithi123 ceph-mon[67226]: Reconfiguring daemon osd.7 on smithi177 2024-04-12T21:51:36.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:35 smithi177 ceph-mon[52869]: pgmap v38: 97 pgs: 97 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.5 MiB/s wr, 615 op/s 2024-04-12T21:51:36.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:35 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:36.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:35 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:36.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:35 smithi177 ceph-mon[52869]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T21:51:36.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:35 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T21:51:36.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:35 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:36.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:35 smithi177 ceph-mon[52869]: Reconfiguring daemon osd.7 on smithi177 2024-04-12T21:51:36.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 186652 11.35 MB/sec execute 256 sec latency 48.963 ms 2024-04-12T21:51:37.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 187261 11.37 MB/sec execute 257 sec latency 30.016 ms 2024-04-12T21:51:37.670 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:37 smithi177 ceph-mon[52869]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 872 op/s 2024-04-12T21:51:37.671 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:37.671 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:37.671 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:37 smithi177 ceph-mon[52869]: Reconfiguring mds.foofs.smithi177.hqctzz (monmap changed)... 2024-04-12T21:51:37.671 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi177.hqctzz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:51:37.671 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:37.671 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:37 smithi177 ceph-mon[52869]: Reconfiguring daemon mds.foofs.smithi177.hqctzz on smithi177 2024-04-12T21:51:37.854 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:37 smithi123 ceph-mon[67226]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 872 op/s 2024-04-12T21:51:37.854 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:37.854 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:37.854 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:37 smithi123 ceph-mon[67226]: Reconfiguring mds.foofs.smithi177.hqctzz (monmap changed)... 2024-04-12T21:51:37.854 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi177.hqctzz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:51:37.854 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:37.854 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:37 smithi123 ceph-mon[67226]: Reconfiguring daemon mds.foofs.smithi177.hqctzz on smithi177 2024-04-12T21:51:38.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 187593 11.37 MB/sec execute 258 sec latency 34.701 ms 2024-04-12T21:51:39.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 187926 11.38 MB/sec execute 259 sec latency 36.663 ms 2024-04-12T21:51:39.627 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all mon 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi123"}]: dispatch 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi123"}]': finished 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi177"}]: dispatch 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi177"}]': finished 2024-04-12T21:51:39.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:39 smithi123 ceph-mon[67226]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.8 MiB/s wr, 602 op/s 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all mon 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi123"}]: dispatch 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi123"}]': finished 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi177"}]: dispatch 2024-04-12T21:51:40.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi177"}]': finished 2024-04-12T21:51:40.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:39 smithi177 ceph-mon[52869]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.8 MiB/s wr, 602 op/s 2024-04-12T21:51:40.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 188294 11.38 MB/sec execute 260 sec latency 42.563 ms 2024-04-12T21:51:41.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 188627 11.37 MB/sec execute 261 sec latency 38.794 ms 2024-04-12T21:51:41.898 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:41 smithi123 ceph-mon[67226]: pgmap v41: 97 pgs: 97 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.8 MiB/s wr, 602 op/s 2024-04-12T21:51:41.898 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:41.898 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:51:41.898 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:42.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:41 smithi177 ceph-mon[52869]: pgmap v41: 97 pgs: 97 active+clean; 330 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.8 MiB/s wr, 602 op/s 2024-04-12T21:51:42.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:42.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:51:42.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:42.521 INFO:teuthology.orchestra.run.smithi123.stdout: 5 189040 11.36 MB/sec execute 262 sec latency 50.538 ms 2024-04-12T21:51:42.914 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:42 smithi123 ceph-mon[67226]: Upgrade: Updating crash.smithi123 (1/2) 2024-04-12T21:51:42.915 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:42 smithi123 ceph-mon[67226]: Deploying daemon crash.smithi123 on smithi123 2024-04-12T21:51:42.931 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:42 smithi177 ceph-mon[52869]: Upgrade: Updating crash.smithi123 (1/2) 2024-04-12T21:51:42.931 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:42 smithi177 ceph-mon[52869]: Deploying daemon crash.smithi123 on smithi123 2024-04-12T21:51:43.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 189522 11.34 MB/sec execute 263 sec latency 41.188 ms 2024-04-12T21:51:43.936 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:43 smithi123 ceph-mon[67226]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 843 op/s 2024-04-12T21:51:43.946 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:43 smithi177 ceph-mon[52869]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 843 op/s 2024-04-12T21:51:44.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 189910 11.35 MB/sec execute 264 sec latency 68.851 ms 2024-04-12T21:51:45.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 190358 11.35 MB/sec execute 265 sec latency 69.800 ms 2024-04-12T21:51:45.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:45 smithi123 ceph-mon[67226]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 497 op/s 2024-04-12T21:51:45.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:51:46.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:45 smithi177 ceph-mon[52869]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 497 op/s 2024-04-12T21:51:46.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:51:46.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 190853 11.35 MB/sec execute 266 sec latency 41.447 ms 2024-04-12T21:51:47.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 191205 11.36 MB/sec execute 267 sec latency 37.552 ms 2024-04-12T21:51:47.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:47 smithi123 ceph-mon[67226]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 767 op/s 2024-04-12T21:51:48.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:47 smithi177 ceph-mon[52869]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 767 op/s 2024-04-12T21:51:48.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 191664 11.37 MB/sec execute 268 sec latency 31.892 ms 2024-04-12T21:51:49.178 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:49 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:49.178 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:49 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:49.178 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:49 smithi123 ceph-mon[67226]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 511 op/s 2024-04-12T21:51:49.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 191982 11.37 MB/sec execute 269 sec latency 55.569 ms 2024-04-12T21:51:49.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:49 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:49.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:49 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:49.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:49 smithi177 ceph-mon[52869]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 511 op/s 2024-04-12T21:51:50.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 192184 11.35 MB/sec execute 270 sec latency 56.095 ms 2024-04-12T21:51:51.260 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:51 smithi123 ceph-mon[67226]: Upgrade: Updating crash.smithi177 (2/2) 2024-04-12T21:51:51.260 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:51 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:51.260 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:51 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:51:51.260 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:51 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:51.260 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:51 smithi123 ceph-mon[67226]: Deploying daemon crash.smithi177 on smithi177 2024-04-12T21:51:51.260 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:51 smithi123 ceph-mon[67226]: pgmap v46: 97 pgs: 97 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 511 op/s 2024-04-12T21:51:51.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:51 smithi177 ceph-mon[52869]: Upgrade: Updating crash.smithi177 (2/2) 2024-04-12T21:51:51.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:51 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:51.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:51 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi177", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T21:51:51.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:51 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:51:51.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:51 smithi177 ceph-mon[52869]: Deploying daemon crash.smithi177 on smithi177 2024-04-12T21:51:51.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:51 smithi177 ceph-mon[52869]: pgmap v46: 97 pgs: 97 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 511 op/s 2024-04-12T21:51:51.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 192373 11.32 MB/sec execute 271 sec latency 72.832 ms 2024-04-12T21:51:52.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 192673 11.29 MB/sec execute 272 sec latency 64.256 ms 2024-04-12T21:51:53.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 192969 11.27 MB/sec execute 273 sec latency 60.141 ms 2024-04-12T21:51:53.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:53 smithi177 ceph-mon[52869]: pgmap v47: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 722 op/s 2024-04-12T21:51:53.792 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:53 smithi123 ceph-mon[67226]: pgmap v47: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 722 op/s 2024-04-12T21:51:54.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 193239 11.26 MB/sec execute 274 sec latency 41.579 ms 2024-04-12T21:51:55.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:55.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:55.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:51:55.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:55 smithi177 ceph-mon[52869]: pgmap v48: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 481 op/s 2024-04-12T21:51:55.338 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:55.338 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:55.338 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:51:55.338 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:55 smithi123 ceph-mon[67226]: pgmap v48: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 481 op/s 2024-04-12T21:51:55.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 193486 11.25 MB/sec execute 275 sec latency 73.046 ms 2024-04-12T21:51:55.886 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:51:56.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:56 smithi177 ceph-mon[52869]: from='client.34174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:56.365 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:56 smithi123 ceph-mon[67226]: from='client.34174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:56.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 193803 11.24 MB/sec execute 276 sec latency 85.634 ms 2024-04-12T21:51:56.998 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:51:56.998 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (3m) 75s ago 13m 25.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:51:56.998 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 starting - - - - 2024-04-12T21:51:56.998 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 starting - - - - 2024-04-12T21:51:56.998 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 unknown 75s ago 12m - - 2024-04-12T21:51:56.998 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (7m) 75s ago 7m 38.5M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:51:56.998 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (7m) 61s ago 7m 14.5M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:51:56.998 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (2m) 75s ago 14m 479M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (117s) 61s ago 12m 448M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (98s) 75s ago 14m 30.6M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (66s) 61s ago 12m 24.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (6m) 75s ago 6m 256M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (6m) 61s ago 6m 84.5M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (3m) 75s ago 12m 21.7M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (3m) 61s ago 12m 18.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (11m) 75s ago 11m 573M 1936M 17.2.0 e1d6a67b021e fc7ca53bb198 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (11m) 75s ago 11m 482M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (10m) 75s ago 10m 349M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (10m) 75s ago 10m 493M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (9m) 61s ago 9m 538M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (9m) 61s ago 9m 372M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (9m) 61s ago 9m 556M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:51:56.999 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (8m) 61s ago 8m 473M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:51:57.000 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (2m) 75s ago 11m 56.1M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:51:57.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:57 smithi177 ceph-mon[52869]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:57.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:57 smithi177 ceph-mon[52869]: pgmap v49: 97 pgs: 97 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 659 op/s 2024-04-12T21:51:57.393 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:57 smithi123 ceph-mon[67226]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:57.394 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:57 smithi123 ceph-mon[67226]: pgmap v49: 97 pgs: 97 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 659 op/s 2024-04-12T21:51:57.524 INFO:teuthology.orchestra.run.smithi123.stdout: 5 194201 11.23 MB/sec execute 277 sec latency 86.056 ms 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T21:51:57.710 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:51:57.711 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:51:57.711 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:51:57.711 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:51:57.711 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:51:57.711 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-12T21:51:57.711 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T21:51:57.711 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:51:57.711 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:51:58.277 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:51:58.277 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:51:58.277 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:51:58.277 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:51:58.278 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:51:58.278 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:51:58.278 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-04-12T21:51:58.278 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:51:58.278 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "5/23 daemons upgraded", 2024-04-12T21:51:58.278 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading crash daemons", 2024-04-12T21:51:58.278 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:51:58.278 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:51:58.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:58 smithi123 ceph-mon[67226]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:58.401 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:58 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1193264219' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:51:58.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 194510 11.22 MB/sec execute 278 sec latency 34.512 ms 2024-04-12T21:51:58.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:58 smithi177 ceph-mon[52869]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:58.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:58 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1193264219' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:51:58.938 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:51:58.938 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:51:58.938 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in unknown state 2024-04-12T21:51:59.406 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:59 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:59.406 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:59 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:59.406 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:59 smithi123 ceph-mon[67226]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:59.406 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:59 smithi123 ceph-mon[67226]: pgmap v50: 97 pgs: 97 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-04-12T21:51:59.406 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:51:59 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3873970665' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:51:59.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 194875 11.23 MB/sec execute 279 sec latency 38.451 ms 2024-04-12T21:51:59.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:59 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:59.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:59 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:51:59.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:59 smithi177 ceph-mon[52869]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:51:59.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:59 smithi177 ceph-mon[52869]: pgmap v50: 97 pgs: 97 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-04-12T21:51:59.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:51:59 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3873970665' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:52:00.412 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:00.412 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:00.412 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:52:00.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 195294 11.23 MB/sec execute 280 sec latency 35.064 ms 2024-04-12T21:52:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:00.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:00.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:52:01.446 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:01 smithi123 ceph-mon[67226]: pgmap v51: 97 pgs: 97 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-04-12T21:52:01.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 195564 11.23 MB/sec execute 281 sec latency 58.169 ms 2024-04-12T21:52:01.740 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:01 smithi177 ceph-mon[52869]: pgmap v51: 97 pgs: 97 active+clean; 324 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-04-12T21:52:02.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 195794 11.22 MB/sec execute 282 sec latency 52.456 ms 2024-04-12T21:52:03.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:03 smithi123 ceph-mon[67226]: mgrmap e42: smithi123.psjrfs(active, since 92s), standbys: smithi177.whybza 2024-04-12T21:52:03.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:03.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:03.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:03 smithi123 ceph-mon[67226]: pgmap v52: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 591 op/s 2024-04-12T21:52:03.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:03.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:03.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 196100 11.19 MB/sec execute 283 sec latency 58.941 ms 2024-04-12T21:52:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:03 smithi177 ceph-mon[52869]: mgrmap e42: smithi123.psjrfs(active, since 92s), standbys: smithi177.whybza 2024-04-12T21:52:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:03 smithi177 ceph-mon[52869]: pgmap v52: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 591 op/s 2024-04-12T21:52:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:04.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 196542 11.17 MB/sec execute 284 sec latency 35.717 ms 2024-04-12T21:52:04.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:04.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:05.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:05.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:05.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 196885 11.16 MB/sec execute 285 sec latency 36.614 ms 2024-04-12T21:52:06.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:05 smithi123 ceph-mon[67226]: pgmap v53: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.7 MiB/s wr, 379 op/s 2024-04-12T21:52:06.164 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:05 smithi177 ceph-mon[52869]: pgmap v53: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.7 MiB/s wr, 379 op/s 2024-04-12T21:52:06.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 197383 11.19 MB/sec execute 286 sec latency 46.831 ms 2024-04-12T21:52:07.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 197925 11.19 MB/sec execute 287 sec latency 58.172 ms 2024-04-12T21:52:08.055 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:07 smithi123 ceph-mon[67226]: pgmap v54: 97 pgs: 97 active+clean; 337 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 598 op/s 2024-04-12T21:52:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:07 smithi177 ceph-mon[52869]: pgmap v54: 97 pgs: 97 active+clean; 337 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 598 op/s 2024-04-12T21:52:08.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 198276 11.19 MB/sec execute 288 sec latency 34.538 ms 2024-04-12T21:52:09.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 198786 11.20 MB/sec execute 289 sec latency 43.955 ms 2024-04-12T21:52:10.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:09 smithi123 ceph-mon[67226]: pgmap v55: 97 pgs: 97 active+clean; 337 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 420 op/s 2024-04-12T21:52:10.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:10.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:10.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:52:10.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:52:10.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:09 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:10.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:09 smithi177 ceph-mon[52869]: pgmap v55: 97 pgs: 97 active+clean; 337 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 420 op/s 2024-04-12T21:52:10.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:10.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:10.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:52:10.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:52:10.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:09 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:10.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 199171 11.21 MB/sec execute 290 sec latency 70.862 ms 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: pgmap v56: 97 pgs: 97 active+clean; 337 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.6 MiB/s wr, 449 op/s 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi123"}]: dispatch 2024-04-12T21:52:11.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi123"}]': finished 2024-04-12T21:52:11.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi177"}]: dispatch 2024-04-12T21:52:11.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi177"}]': finished 2024-04-12T21:52:11.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: pgmap v56: 97 pgs: 97 active+clean; 337 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.6 MiB/s wr, 449 op/s 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi123"}]: dispatch 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi123"}]': finished 2024-04-12T21:52:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi177"}]: dispatch 2024-04-12T21:52:11.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi177"}]': finished 2024-04-12T21:52:11.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T21:52:11.526 INFO:teuthology.orchestra.run.smithi123.stdout: 5 199458 11.20 MB/sec execute 291 sec latency 41.820 ms 2024-04-12T21:52:12.113 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:11 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all crash 2024-04-12T21:52:12.114 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:11 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T21:52:12.114 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:11 smithi123 ceph-mon[67226]: Upgrade: osd.0 is safe to restart 2024-04-12T21:52:12.182 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:11 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all crash 2024-04-12T21:52:12.182 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:11 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T21:52:12.182 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:11 smithi177 ceph-mon[52869]: Upgrade: osd.0 is safe to restart 2024-04-12T21:52:12.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 199843 11.18 MB/sec execute 292 sec latency 68.000 ms 2024-04-12T21:52:12.888 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:12 smithi123 ceph-mon[67226]: pgmap v57: 97 pgs: 97 active+clean; 338 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.7 MiB/s wr, 733 op/s 2024-04-12T21:52:12.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:12 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:12.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:12 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T21:52:12.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:12 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:52:13.198 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:12 smithi177 ceph-mon[52869]: pgmap v57: 97 pgs: 97 active+clean; 338 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.7 MiB/s wr, 733 op/s 2024-04-12T21:52:13.198 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:12 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:13.198 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:12 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T21:52:13.198 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:12 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:52:13.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 200124 11.15 MB/sec execute 293 sec latency 45.332 ms 2024-04-12T21:52:13.917 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:13 smithi123 ceph-mon[67226]: Upgrade: Updating osd.0 2024-04-12T21:52:13.917 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:13 smithi123 ceph-mon[67226]: Deploying daemon osd.0 on smithi123 2024-04-12T21:52:14.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:13 smithi177 ceph-mon[52869]: Upgrade: Updating osd.0 2024-04-12T21:52:14.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:13 smithi177 ceph-mon[52869]: Deploying daemon osd.0 on smithi123 2024-04-12T21:52:14.522 INFO:teuthology.orchestra.run.smithi123.stdout: 5 200382 11.14 MB/sec execute 294 sec latency 62.969 ms 2024-04-12T21:52:14.938 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:14 smithi123 ceph-mon[67226]: pgmap v58: 97 pgs: 97 active+clean; 338 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-04-12T21:52:15.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:14 smithi177 ceph-mon[52869]: pgmap v58: 97 pgs: 97 active+clean; 338 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-04-12T21:52:15.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 200735 11.15 MB/sec execute 295 sec latency 92.247 ms 2024-04-12T21:52:15.954 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:52:16.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:52:16.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 201110 11.15 MB/sec execute 296 sec latency 55.602 ms 2024-04-12T21:52:16.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:16 smithi123 ceph-mon[67226]: pgmap v59: 97 pgs: 97 active+clean; 338 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-04-12T21:52:17.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:16 smithi177 ceph-mon[52869]: pgmap v59: 97 pgs: 97 active+clean; 338 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-04-12T21:52:17.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 201459 11.14 MB/sec execute 297 sec latency 43.348 ms 2024-04-12T21:52:17.999 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:17 smithi123 systemd[1]: Stopping Ceph osd.0 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:52:18.271 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:18 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[34980]: 2024-04-12T21:52:18.113+0000 7f23e147a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:52:18.271 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:18 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[34980]: 2024-04-12T21:52:18.113+0000 7f23e147a700 -1 osd.0 72 *** Got signal Terminated *** 2024-04-12T21:52:18.271 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:18 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[34980]: 2024-04-12T21:52:18.113+0000 7f23e147a700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T21:52:18.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 201661 11.12 MB/sec execute 298 sec latency 46.296 ms 2024-04-12T21:52:19.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:18 smithi123 ceph-mon[67226]: pgmap v60: 97 pgs: 97 active+clean; 331 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.0 MiB/s wr, 505 op/s 2024-04-12T21:52:19.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:18 smithi123 ceph-mon[67226]: osd.0 marked itself down and dead 2024-04-12T21:52:19.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:18 smithi177 ceph-mon[52869]: pgmap v60: 97 pgs: 97 active+clean; 331 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.0 MiB/s wr, 505 op/s 2024-04-12T21:52:19.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:18 smithi177 ceph-mon[52869]: osd.0 marked itself down and dead 2024-04-12T21:52:19.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 201794 11.09 MB/sec execute 299 sec latency 642.871 ms 2024-04-12T21:52:19.824 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:19 smithi123 podman[73128]: 2024-04-12 21:52:19.563158241 +0000 UTC m=+1.625962368 container died fc7ca53bb1983f4edfc93bf4d5bec3ace767beef89e9e221a04a7adeb16d357b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, GIT_CLEAN=True, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, distribution-scope=public, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream) 2024-04-12T21:52:19.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:19 smithi123 ceph-mon[67226]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:52:19.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:19 smithi123 ceph-mon[67226]: osdmap e73: 8 total, 7 up, 8 in 2024-04-12T21:52:20.075 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:20 smithi123 podman[73128]: 2024-04-12 21:52:20.072417302 +0000 UTC m=+2.135221446 container cleanup fc7ca53bb1983f4edfc93bf4d5bec3ace767beef89e9e221a04a7adeb16d357b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, name=centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.openshift.expose-services=, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754) 2024-04-12T21:52:20.075 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:20 smithi123 bash[73128]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0 2024-04-12T21:52:20.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:19 smithi177 ceph-mon[52869]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:52:20.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:19 smithi177 ceph-mon[52869]: osdmap e73: 8 total, 7 up, 8 in 2024-04-12T21:52:20.470 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:20 smithi123 podman[73144]: 2024-04-12 21:52:20.407503021 +0000 UTC m=+0.839404662 container remove fc7ca53bb1983f4edfc93bf4d5bec3ace767beef89e9e221a04a7adeb16d357b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, release=754, build-date=2022-05-03T08:36:31.336870, version=8, GIT_CLEAN=True, ceph=True) 2024-04-12T21:52:20.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 201948 11.08 MB/sec execute 300 sec latency 1058.045 ms 2024-04-12T21:52:20.973 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:20 smithi123 podman[73237]: 2024-04-12 21:52:20.80414672 +0000 UTC m=+0.023241468 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:52:20.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:20 smithi123 ceph-mon[67226]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 331 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-04-12T21:52:20.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:20 smithi123 ceph-mon[67226]: osdmap e74: 8 total, 7 up, 8 in 2024-04-12T21:52:21.229 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:20 smithi123 podman[73237]: 2024-04-12 21:52:20.97308878 +0000 UTC m=+0.192183494 container create 2fd601b7ee48eed6464d59cb10a8fa856e78c14f5dfa68cc1e01f212724c17e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:52:21.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:20 smithi177 ceph-mon[52869]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 331 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-04-12T21:52:21.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:20 smithi177 ceph-mon[52869]: osdmap e74: 8 total, 7 up, 8 in 2024-04-12T21:52:21.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 202255 11.07 MB/sec execute 301 sec latency 101.092 ms 2024-04-12T21:52:21.743 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:21 smithi123 podman[73237]: 2024-04-12 21:52:21.568527243 +0000 UTC m=+0.787621972 container init 2fd601b7ee48eed6464d59cb10a8fa856e78c14f5dfa68cc1e01f212724c17e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-04-12T21:52:21.743 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:21 smithi123 podman[73237]: 2024-04-12 21:52:21.57925913 +0000 UTC m=+0.798353851 container start 2fd601b7ee48eed6464d59cb10a8fa856e78c14f5dfa68cc1e01f212724c17e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-12T21:52:21.996 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:21 smithi123 ceph-mon[67226]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 524 op/s; 211/1635 objects degraded (12.905%) 2024-04-12T21:52:21.996 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:21 smithi123 ceph-mon[67226]: Health check failed: Degraded data redundancy: 211/1635 objects degraded (12.905%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:21.996 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:21 smithi123 podman[73237]: 2024-04-12 21:52:21.756902847 +0000 UTC m=+0.975997573 container attach 2fd601b7ee48eed6464d59cb10a8fa856e78c14f5dfa68cc1e01f212724c17e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:52:21.996 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:21 smithi123 podman[73237]: 2024-04-12 21:52:21.954681685 +0000 UTC m=+1.173776407 container died 2fd601b7ee48eed6464d59cb10a8fa856e78c14f5dfa68cc1e01f212724c17e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb) 2024-04-12T21:52:22.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:21 smithi177 ceph-mon[52869]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 524 op/s; 211/1635 objects degraded (12.905%) 2024-04-12T21:52:22.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:21 smithi177 ceph-mon[52869]: Health check failed: Degraded data redundancy: 211/1635 objects degraded (12.905%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:22.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 202551 11.07 MB/sec execute 302 sec latency 52.100 ms 2024-04-12T21:52:23.057 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:22 smithi123 podman[73263]: 2024-04-12 21:52:22.888949173 +0000 UTC m=+0.928310053 container remove 2fd601b7ee48eed6464d59cb10a8fa856e78c14f5dfa68cc1e01f212724c17e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_CLEAN=True) 2024-04-12T21:52:23.057 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:22 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-12T21:52:23.057 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:22 smithi123 systemd[1]: Stopped Ceph osd.0 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:52:23.057 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:22 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.0.service: Consumed 3min 42.026s CPU time. 2024-04-12T21:52:23.307 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:23 smithi123 systemd[1]: Starting Ceph osd.0 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:52:23.523 INFO:teuthology.orchestra.run.smithi123.stdout: 5 202790 11.06 MB/sec execute 303 sec latency 53.136 ms 2024-04-12T21:52:23.559 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:23 smithi123 podman[73365]: 2024-04-12 21:52:23.355864748 +0000 UTC m=+0.021831954 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:52:23.560 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:23 smithi123 podman[73365]: 2024-04-12 21:52:23.456013382 +0000 UTC m=+0.121980573 container create 2e7019efea65e814289f92fff0f2de0df39d1653b7aec46ca27c2b016d4a0d1d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-12T21:52:23.810 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:23 smithi123 podman[73365]: 2024-04-12 21:52:23.738849956 +0000 UTC m=+0.404817159 container init 2e7019efea65e814289f92fff0f2de0df39d1653b7aec46ca27c2b016d4a0d1d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-12T21:52:23.810 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:23 smithi123 podman[73365]: 2024-04-12 21:52:23.745253077 +0000 UTC m=+0.411220273 container start 2e7019efea65e814289f92fff0f2de0df39d1653b7aec46ca27c2b016d4a0d1d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-12T21:52:24.061 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:23 smithi123 podman[73365]: 2024-04-12 21:52:23.867199188 +0000 UTC m=+0.533166393 container attach 2e7019efea65e814289f92fff0f2de0df39d1653b7aec46ca27c2b016d4a0d1d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-12T21:52:24.531 INFO:teuthology.orchestra.run.smithi123.stdout: 5 203326 11.05 MB/sec execute 304 sec latency 34.201 ms 2024-04-12T21:52:24.565 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate[73380]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate[73380]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 bash[73365]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 bash[73365]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 bash[73365]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 bash[73365]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 bash[73365]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 bash[73365]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 bash[73365]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate[73380]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate[73380]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T21:52:24.566 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate[73380]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-12T21:52:24.567 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate[73380]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T21:52:24.567 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate[73380]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-12T21:52:24.567 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:24 smithi123 podman[73365]: 2024-04-12 21:52:24.466355207 +0000 UTC m=+1.132322406 container died 2e7019efea65e814289f92fff0f2de0df39d1653b7aec46ca27c2b016d4a0d1d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T21:52:25.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:24 smithi177 ceph-mon[52869]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 524 op/s; 211/1635 objects degraded (12.905%) 2024-04-12T21:52:25.073 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:24 smithi123 ceph-mon[67226]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 524 op/s; 211/1635 objects degraded (12.905%) 2024-04-12T21:52:25.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 203994 11.06 MB/sec execute 305 sec latency 17.113 ms 2024-04-12T21:52:25.577 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:25 smithi123 podman[73511]: 2024-04-12 21:52:25.572290676 +0000 UTC m=+1.098361119 container remove 2e7019efea65e814289f92fff0f2de0df39d1653b7aec46ca27c2b016d4a0d1d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T21:52:26.084 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:25 smithi123 podman[73597]: 2024-04-12 21:52:25.767836588 +0000 UTC m=+0.035726990 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:52:26.084 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:25 smithi123 podman[73597]: 2024-04-12 21:52:25.966713215 +0000 UTC m=+0.234603581 container create 8555f56fa001d0eeeed11b159e7ec1068fd5de3891e9c2dcca92c38c2dee4f60 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-04-12T21:52:26.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 204827 11.10 MB/sec execute 306 sec latency 24.381 ms 2024-04-12T21:52:26.586 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:26 smithi123 podman[73597]: 2024-04-12 21:52:26.344828037 +0000 UTC m=+0.612718410 container init 8555f56fa001d0eeeed11b159e7ec1068fd5de3891e9c2dcca92c38c2dee4f60 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-12T21:52:26.586 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:26 smithi123 podman[73597]: 2024-04-12 21:52:26.35186903 +0000 UTC m=+0.619759402 container start 8555f56fa001d0eeeed11b159e7ec1068fd5de3891e9c2dcca92c38c2dee4f60 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-12T21:52:26.586 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:26 smithi123 bash[73597]: 8555f56fa001d0eeeed11b159e7ec1068fd5de3891e9c2dcca92c38c2dee4f60 2024-04-12T21:52:26.586 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:26 smithi123 systemd[1]: Started Ceph osd.0 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:52:27.097 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:26 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[73612]: 2024-04-12T21:52:26.936+0000 7f1c196927c0 -1 Falling back to public interface 2024-04-12T21:52:27.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:26 smithi123 ceph-mon[67226]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.2 KiB/s rd, 2.2 MiB/s wr, 213 op/s; 211/1635 objects degraded (12.905%) 2024-04-12T21:52:27.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:27.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:27.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:52:27.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:26 smithi177 ceph-mon[52869]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.2 KiB/s rd, 2.2 MiB/s wr, 213 op/s; 211/1635 objects degraded (12.905%) 2024-04-12T21:52:27.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:27.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:27.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:52:27.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 205272 11.10 MB/sec execute 307 sec latency 45.510 ms 2024-04-12T21:52:28.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:27 smithi123 ceph-mon[67226]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.8 MiB/s wr, 559 op/s; 246/1917 objects degraded (12.833%) 2024-04-12T21:52:28.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:27 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 246/1917 objects degraded (12.833%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:27 smithi177 ceph-mon[52869]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.8 MiB/s wr, 559 op/s; 246/1917 objects degraded (12.833%) 2024-04-12T21:52:28.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:27 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 246/1917 objects degraded (12.833%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:28.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 205815 11.12 MB/sec execute 308 sec latency 30.647 ms 2024-04-12T21:52:29.464 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:52:29.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 206208 11.13 MB/sec execute 309 sec latency 54.221 ms 2024-04-12T21:52:30.533 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:52:30.533 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (4m) 28s ago 13m 25.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:52:30.533 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (43s) 28s ago 13m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (36s) 32s ago 12m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 28s ago 13m - - 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (7m) 28s ago 7m 38.2M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (7m) 32s ago 7m 14.7M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (3m) 28s ago 14m 503M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (2m) 32s ago 12m 450M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (2m) 28s ago 15m 42.9M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (100s) 32s ago 12m 34.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (7m) 28s ago 7m 258M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (7m) 32s ago 7m 84.7M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (4m) 28s ago 13m 19.9M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (3m) 32s ago 12m 19.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 starting - - - 1936M 2024-04-12T21:52:30.534 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (11m) 28s ago 11m 577M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:52:30.535 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (11m) 28s ago 11m 469M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:52:30.535 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (10m) 28s ago 10m 588M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:52:30.535 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (10m) 32s ago 10m 561M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:52:30.535 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (10m) 32s ago 10m 439M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:52:30.535 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (9m) 32s ago 9m 347M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:52:30.535 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (9m) 32s ago 9m 564M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:52:30.535 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (2m) 28s ago 12m 66.9M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:52:30.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 206530 11.13 MB/sec execute 310 sec latency 54.688 ms 2024-04-12T21:52:30.908 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:30 smithi123 ceph-mon[67226]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:52:30.909 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:30 smithi123 ceph-mon[67226]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.2 MiB/s wr, 500 op/s; 246/1917 objects degraded (12.833%) 2024-04-12T21:52:30.909 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:30.909 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:52:31.166 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:52:31.166 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:52:31.169 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:52:31.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:52:31.169 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:52:31.169 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:52:31.169 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:52:31.169 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:52:31.169 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:52:31.170 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:52:31.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:30 smithi177 ceph-mon[52869]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:52:31.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:30 smithi177 ceph-mon[52869]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 340 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.2 MiB/s wr, 500 op/s; 246/1917 objects degraded (12.833%) 2024-04-12T21:52:31.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:31.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:52:31.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 206982 11.11 MB/sec execute 311 sec latency 44.019 ms 2024-04-12T21:52:31.689 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:52:31.689 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:52:31.689 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:52:31.689 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "6/23 daemons upgraded", 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:52:31.690 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:52:31.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:31 smithi123 ceph-mon[67226]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:52:31.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:31 smithi123 ceph-mon[67226]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:52:31.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:31 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1501180959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:32.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:31 smithi177 ceph-mon[52869]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:52:32.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:31 smithi177 ceph-mon[52869]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:52:32.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:31 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1501180959' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:32.341 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 228/1773 objects degraded (12.860%), 29 pgs degraded 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout: osd.0 (root=default,host=smithi123) is down 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 228/1773 objects degraded (12.860%), 29 pgs degraded 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-04-12T21:52:32.342 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-04-12T21:52:32.343 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-04-12T21:52:32.344 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-04-12T21:52:32.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 207363 11.10 MB/sec execute 312 sec latency 36.696 ms 2024-04-12T21:52:32.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:32 smithi123 ceph-mon[67226]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:52:32.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:32 smithi123 ceph-mon[67226]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 342 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.3 MiB/s wr, 708 op/s; 228/1773 objects degraded (12.860%) 2024-04-12T21:52:32.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:32 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3949601497' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:52:33.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:32 smithi177 ceph-mon[52869]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:52:33.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:32 smithi177 ceph-mon[52869]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 342 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.3 MiB/s wr, 708 op/s; 228/1773 objects degraded (12.860%) 2024-04-12T21:52:33.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:32 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3949601497' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:52:33.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 207884 11.13 MB/sec execute 313 sec latency 30.908 ms 2024-04-12T21:52:34.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 208483 11.14 MB/sec execute 314 sec latency 45.999 ms 2024-04-12T21:52:34.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:34 smithi123 ceph-mon[67226]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 342 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 562 op/s; 228/1773 objects degraded (12.860%) 2024-04-12T21:52:34.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:34 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 228/1773 objects degraded (12.860%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:34.957 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:34 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[73612]: 2024-04-12T21:52:34.725+0000 7f1c196927c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-12T21:52:35.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:34 smithi177 ceph-mon[52869]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 342 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 562 op/s; 228/1773 objects degraded (12.860%) 2024-04-12T21:52:35.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:34 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 228/1773 objects degraded (12.860%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:35.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 208777 11.12 MB/sec execute 315 sec latency 49.988 ms 2024-04-12T21:52:36.504 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:36 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[73612]: 2024-04-12T21:52:36.384+0000 7f1c196927c0 -1 osd.0 72 log_to_monitors true 2024-04-12T21:52:36.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209070 11.12 MB/sec execute 316 sec latency 69.222 ms 2024-04-12T21:52:36.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:36.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:36.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:36 smithi123 ceph-mon[67226]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 342 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 562 op/s; 228/1773 objects degraded (12.860%) 2024-04-12T21:52:36.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:36 smithi123 ceph-mon[67226]: from='osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T21:52:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:36 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:37.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:36 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:37.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:36 smithi177 ceph-mon[52869]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 342 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 562 op/s; 228/1773 objects degraded (12.860%) 2024-04-12T21:52:37.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:36 smithi177 ceph-mon[52869]: from='osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T21:52:37.283 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 21:52:37 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[73612]: 2024-04-12T21:52:37.254+0000 7f1c06ae2700 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:52:37.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209356 11.11 MB/sec execute 317 sec latency 107.969 ms 2024-04-12T21:52:37.803 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:37 smithi123 ceph-mon[67226]: from='osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T21:52:37.803 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:37 smithi123 ceph-mon[67226]: osdmap e75: 8 total, 7 up, 8 in 2024-04-12T21:52:37.803 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:37 smithi123 ceph-mon[67226]: from='osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:52:37.803 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:37 smithi123 ceph-mon[67226]: from='osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142]' entity='osd.0' 2024-04-12T21:52:37.803 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:37.803 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:38.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:37 smithi177 ceph-mon[52869]: from='osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T21:52:38.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:37 smithi177 ceph-mon[52869]: osdmap e75: 8 total, 7 up, 8 in 2024-04-12T21:52:38.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:37 smithi177 ceph-mon[52869]: from='osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:52:38.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:37 smithi177 ceph-mon[52869]: from='osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142]' entity='osd.0' 2024-04-12T21:52:38.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:38.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:38.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209688 11.11 MB/sec execute 318 sec latency 125.016 ms 2024-04-12T21:52:38.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:38 smithi123 ceph-mon[67226]: pgmap v73: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 667 op/s; 219/1725 objects degraded (12.696%) 2024-04-12T21:52:38.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:38 smithi123 ceph-mon[67226]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:52:38.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:38 smithi123 ceph-mon[67226]: osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142] boot 2024-04-12T21:52:38.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:38 smithi123 ceph-mon[67226]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T21:52:38.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:38 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:52:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:38 smithi177 ceph-mon[52869]: pgmap v73: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 667 op/s; 219/1725 objects degraded (12.696%) 2024-04-12T21:52:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:38 smithi177 ceph-mon[52869]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:52:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:38 smithi177 ceph-mon[52869]: osd.0 [v2:172.21.15.123:6802/2407005142,v1:172.21.15.123:6803/2407005142] boot 2024-04-12T21:52:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:38 smithi177 ceph-mon[52869]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T21:52:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:38 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T21:52:39.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209761 11.09 MB/sec execute 319 sec latency 1125.048 ms 2024-04-12T21:52:40.349 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:40 smithi123 ceph-mon[67226]: osdmap e77: 8 total, 8 up, 8 in 2024-04-12T21:52:40.349 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:40 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 219/1725 objects degraded (12.696%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:40.349 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:40 smithi123 ceph-mon[67226]: pgmap v76: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.2 MiB/s wr, 536 op/s; 219/1725 objects degraded (12.696%) 2024-04-12T21:52:40.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:40 smithi177 ceph-mon[52869]: osdmap e77: 8 total, 8 up, 8 in 2024-04-12T21:52:40.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:40 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 219/1725 objects degraded (12.696%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:40.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:40 smithi177 ceph-mon[52869]: pgmap v76: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.2 MiB/s wr, 536 op/s; 219/1725 objects degraded (12.696%) 2024-04-12T21:52:40.534 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209796 11.06 MB/sec execute 320 sec latency 1980.551 ms 2024-04-12T21:52:41.412 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:41 smithi123 ceph-mon[67226]: osdmap e78: 8 total, 8 up, 8 in 2024-04-12T21:52:41.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:41 smithi177 ceph-mon[52869]: osdmap e78: 8 total, 8 up, 8 in 2024-04-12T21:52:41.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 210009 11.04 MB/sec execute 321 sec latency 88.066 ms 2024-04-12T21:52:42.438 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:42 smithi123 ceph-mon[67226]: osdmap e79: 8 total, 8 up, 8 in 2024-04-12T21:52:42.439 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:42 smithi123 ceph-mon[67226]: pgmap v79: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 380 op/s; 269/1563 objects degraded (17.210%); 1.1 MiB/s, 9 objects/s recovering 2024-04-12T21:52:42.439 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:42.439 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:42.439 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:52:42.439 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:52:42.439 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:42.439 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:42.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:42 smithi177 ceph-mon[52869]: osdmap e79: 8 total, 8 up, 8 in 2024-04-12T21:52:42.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:42 smithi177 ceph-mon[52869]: pgmap v79: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 380 op/s; 269/1563 objects degraded (17.210%); 1.1 MiB/s, 9 objects/s recovering 2024-04-12T21:52:42.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:42.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:42.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:52:42.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:52:42.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:42.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:42.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 210242 11.02 MB/sec execute 322 sec latency 223.648 ms 2024-04-12T21:52:43.464 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: pgmap v80: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 385 op/s; 269/1563 objects degraded (17.210%); 1.1 MiB/s, 10 objects/s recovering 2024-04-12T21:52:43.464 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: pgmap v81: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 517 op/s; 269/1563 objects degraded (17.210%); 1.4 MiB/s, 13 objects/s recovering 2024-04-12T21:52:43.464 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T21:52:43.464 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:52:43.464 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:43.464 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:43.464 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:43.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T21:52:43.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T21:52:43.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:43 smithi123 ceph-mon[67226]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T21:52:43.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: pgmap v80: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 385 op/s; 269/1563 objects degraded (17.210%); 1.1 MiB/s, 10 objects/s recovering 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: pgmap v81: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 517 op/s; 269/1563 objects degraded (17.210%); 1.4 MiB/s, 13 objects/s recovering 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T21:52:43.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:43 smithi177 ceph-mon[52869]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T21:52:43.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 210467 10.99 MB/sec execute 323 sec latency 984.390 ms 2024-04-12T21:52:44.500 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:44 smithi123 ceph-mon[67226]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T21:52:44.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 210691 10.97 MB/sec execute 324 sec latency 593.182 ms 2024-04-12T21:52:44.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:44 smithi177 ceph-mon[52869]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T21:52:45.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 210851 10.94 MB/sec execute 325 sec latency 748.782 ms 2024-04-12T21:52:45.550 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:45 smithi123 ceph-mon[67226]: pgmap v84: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 269/1563 objects degraded (17.210%) 2024-04-12T21:52:45.550 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:45 smithi123 ceph-mon[67226]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T21:52:45.550 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:45 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 269/1563 objects degraded (17.210%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:45.550 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:45.550 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:52:45.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:45 smithi177 ceph-mon[52869]: pgmap v84: 97 pgs: 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 269/1563 objects degraded (17.210%) 2024-04-12T21:52:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:45 smithi177 ceph-mon[52869]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T21:52:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:45 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 269/1563 objects degraded (17.210%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T21:52:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:52:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:52:46.328 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:46 smithi123 ceph-mon[67226]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T21:52:46.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 211109 10.94 MB/sec execute 326 sec latency 236.752 ms 2024-04-12T21:52:46.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:46 smithi177 ceph-mon[52869]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T21:52:47.384 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:47 smithi123 ceph-mon[67226]: pgmap v87: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 90 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.3 MiB/s wr, 401 op/s; 87/1620 objects degraded (5.370%); 0 B/s, 41 objects/s recovering 2024-04-12T21:52:47.384 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:47 smithi123 ceph-mon[67226]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T21:52:47.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 211423 10.94 MB/sec execute 327 sec latency 1025.290 ms 2024-04-12T21:52:47.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:47 smithi177 ceph-mon[52869]: pgmap v87: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 90 active+clean; 334 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.3 MiB/s wr, 401 op/s; 87/1620 objects degraded (5.370%); 0 B/s, 41 objects/s recovering 2024-04-12T21:52:47.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:47 smithi177 ceph-mon[52869]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T21:52:48.412 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:48 smithi123 ceph-mon[67226]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T21:52:48.412 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:48 smithi123 ceph-mon[67226]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T21:52:48.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 211805 10.93 MB/sec execute 328 sec latency 832.184 ms 2024-04-12T21:52:48.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:48 smithi177 ceph-mon[52869]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T21:52:48.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:48 smithi177 ceph-mon[52869]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T21:52:49.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:49 smithi123 ceph-mon[67226]: pgmap v90: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 423 op/s; 5/1617 objects degraded (0.309%); 0 B/s, 64 objects/s recovering 2024-04-12T21:52:49.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 212132 10.92 MB/sec execute 329 sec latency 41.154 ms 2024-04-12T21:52:49.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:49 smithi177 ceph-mon[52869]: pgmap v90: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 423 op/s; 5/1617 objects degraded (0.309%); 0 B/s, 64 objects/s recovering 2024-04-12T21:52:50.437 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:50 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 5/1617 objects degraded (0.309%), 1 pg degraded (PG_DEGRADED) 2024-04-12T21:52:50.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 212500 10.93 MB/sec execute 330 sec latency 53.932 ms 2024-04-12T21:52:50.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:50 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 5/1617 objects degraded (0.309%), 1 pg degraded (PG_DEGRADED) 2024-04-12T21:52:51.454 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:51 smithi123 ceph-mon[67226]: pgmap v92: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.2 MiB/s wr, 344 op/s; 5/1617 objects degraded (0.309%); 0 B/s, 52 objects/s recovering 2024-04-12T21:52:51.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 212891 10.93 MB/sec execute 331 sec latency 44.192 ms 2024-04-12T21:52:51.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:51 smithi177 ceph-mon[52869]: pgmap v92: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.2 MiB/s wr, 344 op/s; 5/1617 objects degraded (0.309%); 0 B/s, 52 objects/s recovering 2024-04-12T21:52:52.470 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:52 smithi123 ceph-mon[67226]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1617 objects degraded (0.309%), 1 pg degraded) 2024-04-12T21:52:52.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 213223 10.94 MB/sec execute 332 sec latency 48.973 ms 2024-04-12T21:52:52.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:52 smithi177 ceph-mon[52869]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1617 objects degraded (0.309%), 1 pg degraded) 2024-04-12T21:52:53.520 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:53 smithi123 ceph-mon[67226]: pgmap v93: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.7 MiB/s wr, 415 op/s; 0 B/s, 18 objects/s recovering 2024-04-12T21:52:53.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 213426 10.92 MB/sec execute 333 sec latency 47.899 ms 2024-04-12T21:52:53.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:53 smithi177 ceph-mon[52869]: pgmap v93: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.7 MiB/s wr, 415 op/s; 0 B/s, 18 objects/s recovering 2024-04-12T21:52:54.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 213664 10.90 MB/sec execute 334 sec latency 40.106 ms 2024-04-12T21:52:55.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 214051 10.89 MB/sec execute 335 sec latency 61.028 ms 2024-04-12T21:52:55.551 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:55 smithi123 ceph-mon[67226]: pgmap v94: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.6 MiB/s wr, 314 op/s; 0 B/s, 14 objects/s recovering 2024-04-12T21:52:55.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:55 smithi177 ceph-mon[52869]: pgmap v94: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.6 MiB/s wr, 314 op/s; 0 B/s, 14 objects/s recovering 2024-04-12T21:52:56.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 214378 10.87 MB/sec execute 336 sec latency 48.320 ms 2024-04-12T21:52:57.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 214732 10.88 MB/sec execute 337 sec latency 47.310 ms 2024-04-12T21:52:57.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:57 smithi123 ceph-mon[67226]: pgmap v95: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 519 op/s; 0 B/s, 2 objects/s recovering 2024-04-12T21:52:57.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:57 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T21:52:57.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:57 smithi177 ceph-mon[52869]: pgmap v95: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 519 op/s; 0 B/s, 2 objects/s recovering 2024-04-12T21:52:57.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:57 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T21:52:58.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 215073 10.86 MB/sec execute 338 sec latency 53.679 ms 2024-04-12T21:52:58.647 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:58 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T21:52:58.647 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:58 smithi123 ceph-mon[67226]: Upgrade: osd.1 is safe to restart 2024-04-12T21:52:58.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:58 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T21:52:58.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:58 smithi177 ceph-mon[52869]: Upgrade: osd.1 is safe to restart 2024-04-12T21:52:59.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 215645 10.87 MB/sec execute 339 sec latency 54.321 ms 2024-04-12T21:52:59.652 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:52:59 smithi123 ceph-mon[67226]: pgmap v96: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 474 op/s; 0 B/s, 2 objects/s recovering 2024-04-12T21:52:59.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:52:59 smithi177 ceph-mon[52869]: pgmap v96: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 474 op/s; 0 B/s, 2 objects/s recovering 2024-04-12T21:53:00.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 215958 10.88 MB/sec execute 340 sec latency 44.125 ms 2024-04-12T21:53:00.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:00 smithi123 ceph-mon[67226]: Upgrade: Updating osd.1 2024-04-12T21:53:00.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:00.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T21:53:00.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:53:00.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:00 smithi123 ceph-mon[67226]: Deploying daemon osd.1 on smithi123 2024-04-12T21:53:00.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:00.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:00.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:53:01.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:00 smithi177 ceph-mon[52869]: Upgrade: Updating osd.1 2024-04-12T21:53:01.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:01.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T21:53:01.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:53:01.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:00 smithi177 ceph-mon[52869]: Deploying daemon osd.1 on smithi123 2024-04-12T21:53:01.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:01.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:01.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:53:01.535 INFO:teuthology.orchestra.run.smithi123.stdout: 5 216378 10.89 MB/sec execute 341 sec latency 46.142 ms 2024-04-12T21:53:01.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:01 smithi123 ceph-mon[67226]: pgmap v97: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 398 op/s; 0 B/s, 1 objects/s recovering 2024-04-12T21:53:02.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:01 smithi177 ceph-mon[52869]: pgmap v97: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 398 op/s; 0 B/s, 1 objects/s recovering 2024-04-12T21:53:02.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 216821 10.90 MB/sec execute 342 sec latency 32.785 ms 2024-04-12T21:53:02.956 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:53:03.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 217068 10.89 MB/sec execute 343 sec latency 111.934 ms 2024-04-12T21:53:03.949 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (5m) 28s ago 14m 24.3M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (76s) 28s ago 14m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (70s) 65s ago 13m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 28s ago 13m - - 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (8m) 28s ago 8m 38.5M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (8m) 65s ago 8m 14.7M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (3m) 28s ago 15m 505M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (3m) 65s ago 13m 450M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (2m) 28s ago 15m 45.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:53:03.950 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (2m) 65s ago 13m 34.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (7m) 28s ago 7m 273M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (7m) 65s ago 7m 84.7M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (4m) 28s ago 13m 19.6M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (4m) 65s ago 13m 19.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (38s) 28s ago 12m 12.8M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (12m) 28s ago 12m 348M 1936M 17.2.0 e1d6a67b021e 3e9e973b400b 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (11m) 28s ago 11m 501M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (11m) 28s ago 11m 351M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (10m) 65s ago 10m 561M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (10m) 65s ago 10m 439M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (10m) 65s ago 10m 347M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:53:03.951 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (9m) 65s ago 9m 564M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:53:03.952 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (3m) 28s ago 13m 67.0M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:53:04.009 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:03 smithi123 ceph-mon[67226]: pgmap v98: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.3 MiB/s wr, 623 op/s; 0 B/s, 1 objects/s recovering 2024-04-12T21:53:04.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:03 smithi177 ceph-mon[52869]: pgmap v98: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.3 MiB/s wr, 623 op/s; 0 B/s, 1 objects/s recovering 2024-04-12T21:53:04.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 217315 10.86 MB/sec execute 344 sec latency 52.543 ms 2024-04-12T21:53:04.707 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:53:04.707 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:53:04.707 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:53:04.707 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:53:04.707 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:53:04.707 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 5 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:53:04.708 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:53:05.033 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:04 smithi123 ceph-mon[67226]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:05.033 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:04 smithi123 ceph-mon[67226]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:05.033 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:04 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/737829826' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:05.275 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:53:05.275 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:53:05.275 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:53:05.275 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:53:05.275 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:53:05.276 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:53:05.276 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:53:05.276 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:53:05.276 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:53:05.276 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "7/23 daemons upgraded", 2024-04-12T21:53:05.276 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:53:05.276 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:53:05.276 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:53:05.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:04 smithi177 ceph-mon[52869]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:04 smithi177 ceph-mon[52869]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:04 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/737829826' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:05.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 217690 10.85 MB/sec execute 345 sec latency 53.901 ms 2024-04-12T21:53:05.910 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:53:05.910 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:53:05.910 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:53:06.046 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:05 smithi123 ceph-mon[67226]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:06.046 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:05 smithi123 ceph-mon[67226]: pgmap v99: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 422 op/s 2024-04-12T21:53:06.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:05 smithi177 ceph-mon[52869]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:05 smithi177 ceph-mon[52869]: pgmap v99: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 422 op/s 2024-04-12T21:53:06.308 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:06 smithi123 systemd[1]: Stopping Ceph osd.1 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:53:06.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 218013 10.83 MB/sec execute 346 sec latency 31.735 ms 2024-04-12T21:53:07.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:06 smithi123 ceph-mon[67226]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:07.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:06 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/4223193211' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:53:07.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:06 smithi123 ceph-mon[67226]: osd.1 marked itself down and dead 2024-04-12T21:53:07.068 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:06 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[38116]: 2024-04-12T21:53:06.833+0000 7fd701d41700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:53:07.068 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:06 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[38116]: 2024-04-12T21:53:06.833+0000 7fd701d41700 -1 osd.1 86 *** Got signal Terminated *** 2024-04-12T21:53:07.068 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:06 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[38116]: 2024-04-12T21:53:06.833+0000 7fd701d41700 -1 osd.1 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T21:53:07.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:06 smithi177 ceph-mon[52869]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:07.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:06 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/4223193211' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:53:07.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:06 smithi177 ceph-mon[52869]: osd.1 marked itself down and dead 2024-04-12T21:53:07.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 218296 10.84 MB/sec execute 347 sec latency 609.789 ms 2024-04-12T21:53:07.884 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:07 smithi123 podman[75860]: 2024-04-12 21:53:07.58739503 +0000 UTC m=+1.104450387 container died 3e9e973b400bcc58b0850fbcd53f36a0d7823de6519b9f5fb454adeaf60a34e5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, RELEASE=HEAD, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-12T21:53:08.081 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:07 smithi123 ceph-mon[67226]: pgmap v100: 97 pgs: 97 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.9 MiB/s wr, 628 op/s 2024-04-12T21:53:08.082 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:07 smithi123 ceph-mon[67226]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:53:08.082 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:07 smithi123 ceph-mon[67226]: osdmap e87: 8 total, 7 up, 8 in 2024-04-12T21:53:08.082 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:07 smithi123 podman[75860]: 2024-04-12 21:53:07.990195496 +0000 UTC m=+1.507250858 container cleanup 3e9e973b400bcc58b0850fbcd53f36a0d7823de6519b9f5fb454adeaf60a34e5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-12T21:53:08.082 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:07 smithi123 bash[75860]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1 2024-04-12T21:53:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:07 smithi177 ceph-mon[52869]: pgmap v100: 97 pgs: 97 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.9 MiB/s wr, 628 op/s 2024-04-12T21:53:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:07 smithi177 ceph-mon[52869]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:53:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:07 smithi177 ceph-mon[52869]: osdmap e87: 8 total, 7 up, 8 in 2024-04-12T21:53:08.404 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:08 smithi123 podman[75877]: 2024-04-12 21:53:08.367764174 +0000 UTC m=+0.784786897 container remove 3e9e973b400bcc58b0850fbcd53f36a0d7823de6519b9f5fb454adeaf60a34e5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, RELEASE=HEAD, GIT_BRANCH=HEAD, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, name=centos-stream, ceph=True, vcs-type=git, io.openshift.expose-services=) 2024-04-12T21:53:08.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 218517 10.82 MB/sec execute 348 sec latency 1042.948 ms 2024-04-12T21:53:08.908 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:08 smithi123 podman[75963]: 2024-04-12 21:53:08.792636259 +0000 UTC m=+0.021883910 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:53:09.159 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:08 smithi123 ceph-mon[67226]: osdmap e88: 8 total, 7 up, 8 in 2024-04-12T21:53:09.159 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:08 smithi123 ceph-mon[67226]: pgmap v103: 97 pgs: 14 stale+active+clean, 83 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.6 MiB/s wr, 668 op/s 2024-04-12T21:53:09.159 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:08 smithi123 podman[75963]: 2024-04-12 21:53:08.918889149 +0000 UTC m=+0.148136785 container create e242a1662489a7c19e4f5f9ae96b8f33220e33b42f456f01463fe7831f452e63 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-12T21:53:09.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:08 smithi177 ceph-mon[52869]: osdmap e88: 8 total, 7 up, 8 in 2024-04-12T21:53:09.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:08 smithi177 ceph-mon[52869]: pgmap v103: 97 pgs: 14 stale+active+clean, 83 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.6 MiB/s wr, 668 op/s 2024-04-12T21:53:09.410 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:09 smithi123 podman[75963]: 2024-04-12 21:53:09.393457685 +0000 UTC m=+0.622705339 container init e242a1662489a7c19e4f5f9ae96b8f33220e33b42f456f01463fe7831f452e63 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-12T21:53:09.410 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:09 smithi123 podman[75963]: 2024-04-12 21:53:09.408567095 +0000 UTC m=+0.637814738 container start e242a1662489a7c19e4f5f9ae96b8f33220e33b42f456f01463fe7831f452e63 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb) 2024-04-12T21:53:09.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 218978 10.82 MB/sec execute 349 sec latency 38.060 ms 2024-04-12T21:53:09.662 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:09 smithi123 podman[75963]: 2024-04-12 21:53:09.632625703 +0000 UTC m=+0.861873355 container attach e242a1662489a7c19e4f5f9ae96b8f33220e33b42f456f01463fe7831f452e63 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:53:09.914 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:09 smithi123 podman[75988]: 2024-04-12 21:53:09.869636395 +0000 UTC m=+0.026426545 container died e242a1662489a7c19e4f5f9ae96b8f33220e33b42f456f01463fe7831f452e63 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2) 2024-04-12T21:53:10.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 219306 10.81 MB/sec execute 350 sec latency 34.972 ms 2024-04-12T21:53:10.956 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:10 smithi123 podman[75988]: 2024-04-12 21:53:10.80753713 +0000 UTC m=+0.964327282 container remove e242a1662489a7c19e4f5f9ae96b8f33220e33b42f456f01463fe7831f452e63 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-04-12T21:53:10.956 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:10 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-12T21:53:10.956 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:10 smithi123 systemd[1]: Stopped Ceph osd.1 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:53:10.956 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:10 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.1.service: Consumed 3min 49.226s CPU time. 2024-04-12T21:53:11.233 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:11 smithi123 systemd[1]: Starting Ceph osd.1 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:53:11.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:11 smithi123 ceph-mon[67226]: pgmap v104: 97 pgs: 14 stale+active+clean, 83 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 MiB/s wr, 326 op/s 2024-04-12T21:53:11.495 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:11 smithi123 podman[76085]: 2024-04-12 21:53:11.299026826 +0000 UTC m=+0.023500182 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:53:11.495 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:11 smithi123 podman[76085]: 2024-04-12 21:53:11.425826873 +0000 UTC m=+0.150300199 container create 218162b212666ac6cf0884eca3a340f76643a8ac6bd1ba614ef3a9c5e32ec185 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-12T21:53:11.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:11 smithi177 ceph-mon[52869]: pgmap v104: 97 pgs: 14 stale+active+clean, 83 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 MiB/s wr, 326 op/s 2024-04-12T21:53:11.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 219606 10.82 MB/sec execute 351 sec latency 87.066 ms 2024-04-12T21:53:11.747 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:11 smithi123 podman[76085]: 2024-04-12 21:53:11.69600833 +0000 UTC m=+0.420481663 container init 218162b212666ac6cf0884eca3a340f76643a8ac6bd1ba614ef3a9c5e32ec185 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-12T21:53:11.747 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:11 smithi123 podman[76085]: 2024-04-12 21:53:11.706372983 +0000 UTC m=+0.430846310 container start 218162b212666ac6cf0884eca3a340f76643a8ac6bd1ba614ef3a9c5e32ec185 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T21:53:11.999 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:11 smithi123 podman[76085]: 2024-04-12 21:53:11.826823942 +0000 UTC m=+0.551297283 container attach 218162b212666ac6cf0884eca3a340f76643a8ac6bd1ba614ef3a9c5e32ec185 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-04-12T21:53:12.516 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:12 smithi123 ceph-mon[67226]: Health check failed: Degraded data redundancy: 186/1695 objects degraded (10.973%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:53:12.516 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate[76100]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T21:53:12.516 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate[76100]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate[76100]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate[76100]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate[76100]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate[76100]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate[76100]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 bash[76085]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 bash[76085]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 bash[76085]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 bash[76085]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 bash[76085]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-12T21:53:12.517 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 bash[76085]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T21:53:12.518 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 bash[76085]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-12T21:53:12.518 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:12 smithi123 podman[76085]: 2024-04-12 21:53:12.322460846 +0000 UTC m=+1.046934188 container died 218162b212666ac6cf0884eca3a340f76643a8ac6bd1ba614ef3a9c5e32ec185 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS) 2024-04-12T21:53:12.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 220006 10.82 MB/sec execute 352 sec latency 49.810 ms 2024-04-12T21:53:12.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:12 smithi177 ceph-mon[52869]: Health check failed: Degraded data redundancy: 186/1695 objects degraded (10.973%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:53:13.527 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:13 smithi123 ceph-mon[67226]: pgmap v105: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.0 MiB/s wr, 617 op/s; 186/1695 objects degraded (10.973%) 2024-04-12T21:53:13.527 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:13 smithi123 podman[76236]: 2024-04-12 21:53:13.360425033 +0000 UTC m=+1.031262135 container remove 218162b212666ac6cf0884eca3a340f76643a8ac6bd1ba614ef3a9c5e32ec185 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-04-12T21:53:13.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 220470 10.84 MB/sec execute 353 sec latency 39.271 ms 2024-04-12T21:53:13.778 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:13 smithi123 podman[76292]: 2024-04-12 21:53:13.519847144 +0000 UTC m=+0.025868818 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:53:13.778 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:13 smithi123 podman[76292]: 2024-04-12 21:53:13.669958397 +0000 UTC m=+0.175980064 container create 5308afb6272c994bac44fe849d714c2c4229c116645d7c26b672221272906f29 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409) 2024-04-12T21:53:13.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:13 smithi177 ceph-mon[52869]: pgmap v105: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.0 MiB/s wr, 617 op/s; 186/1695 objects degraded (10.973%) 2024-04-12T21:53:14.040 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:13 smithi123 podman[76292]: 2024-04-12 21:53:13.931295939 +0000 UTC m=+0.437317622 container init 5308afb6272c994bac44fe849d714c2c4229c116645d7c26b672221272906f29 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.build-date=20240409) 2024-04-12T21:53:14.041 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:13 smithi123 podman[76292]: 2024-04-12 21:53:13.939072769 +0000 UTC m=+0.445094429 container start 5308afb6272c994bac44fe849d714c2c4229c116645d7c26b672221272906f29 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:53:14.041 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:13 smithi123 ceph-osd[76310]: -- 172.21.15.123:0/2185838451 <== mon.0 v2:172.21.15.123:3300/0 3 ==== mon_map magic: 0 ==== 357+0+0 (secure 0 0 0) 0x555b2d913ba0 con 0x555b2d8e4c00 2024-04-12T21:53:14.041 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:13 smithi123 ceph-osd[76310]: -- 172.21.15.123:0/2185838451 <== mon.0 v2:172.21.15.123:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555b2e6b61e0 con 0x555b2d8e4c00 2024-04-12T21:53:14.290 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:14 smithi123 bash[76292]: 5308afb6272c994bac44fe849d714c2c4229c116645d7c26b672221272906f29 2024-04-12T21:53:14.291 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:14 smithi123 systemd[1]: Started Ceph osd.1 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:53:14.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 220790 10.83 MB/sec execute 354 sec latency 48.448 ms 2024-04-12T21:53:14.544 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:14 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T21:53:14.515+0000 7f0ff5fac7c0 -1 Falling back to public interface 2024-04-12T21:53:15.310 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:15.310 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:15.310 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:53:15.310 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:15 smithi123 ceph-mon[67226]: pgmap v106: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.3 MiB/s wr, 309 op/s; 186/1695 objects degraded (10.973%) 2024-04-12T21:53:15.310 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:15.310 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:53:15.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:15.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:15.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:53:15.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:15 smithi177 ceph-mon[52869]: pgmap v106: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.3 MiB/s wr, 309 op/s; 186/1695 objects degraded (10.973%) 2024-04-12T21:53:15.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:15.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:53:15.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 221131 10.81 MB/sec execute 355 sec latency 55.789 ms 2024-04-12T21:53:16.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 221412 10.79 MB/sec execute 356 sec latency 47.329 ms 2024-04-12T21:53:17.356 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:17 smithi123 ceph-mon[67226]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 524 op/s; 177/1644 objects degraded (10.766%) 2024-04-12T21:53:17.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:17 smithi177 ceph-mon[52869]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 524 op/s; 177/1644 objects degraded (10.766%) 2024-04-12T21:53:17.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 221737 10.79 MB/sec execute 357 sec latency 43.152 ms 2024-04-12T21:53:18.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:18 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 177/1671 objects degraded (10.592%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:53:18.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 222116 10.79 MB/sec execute 358 sec latency 48.022 ms 2024-04-12T21:53:18.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:18 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 177/1671 objects degraded (10.592%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:53:19.391 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:19 smithi123 ceph-mon[67226]: pgmap v108: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 492 op/s; 177/1671 objects degraded (10.592%) 2024-04-12T21:53:19.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 222668 10.79 MB/sec execute 359 sec latency 26.403 ms 2024-04-12T21:53:19.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:19 smithi177 ceph-mon[52869]: pgmap v108: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 492 op/s; 177/1671 objects degraded (10.592%) 2024-04-12T21:53:20.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 222971 10.79 MB/sec execute 360 sec latency 29.227 ms 2024-04-12T21:53:21.426 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:21 smithi123 ceph-mon[67226]: pgmap v109: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 421 op/s; 177/1671 objects degraded (10.592%) 2024-04-12T21:53:21.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 223335 10.81 MB/sec execute 361 sec latency 48.858 ms 2024-04-12T21:53:21.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:21 smithi177 ceph-mon[52869]: pgmap v109: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 421 op/s; 177/1671 objects degraded (10.592%) 2024-04-12T21:53:21.928 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:21 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T21:53:21.904+0000 7f0ff5fac7c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-12T21:53:22.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 223683 10.80 MB/sec execute 362 sec latency 71.566 ms 2024-04-12T21:53:23.472 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:23 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T21:53:23.256+0000 7f0ff5fac7c0 -1 osd.1 86 log_to_monitors true 2024-04-12T21:53:23.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224011 10.80 MB/sec execute 363 sec latency 41.611 ms 2024-04-12T21:53:24.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:23 smithi123 ceph-mon[67226]: pgmap v110: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.6 MiB/s wr, 652 op/s; 189/1734 objects degraded (10.900%) 2024-04-12T21:53:24.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:23 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:24.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:23 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:24.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:23 smithi123 ceph-mon[67226]: from='osd.1 [v2:172.21.15.123:6810/3126033631,v1:172.21.15.123:6811/3126033631]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T21:53:24.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:23 smithi123 ceph-mon[67226]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T21:53:24.004 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:53:23 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T21:53:23.942+0000 7f0fe7404700 -1 osd.1 86 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:53:24.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:23 smithi177 ceph-mon[52869]: pgmap v110: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.6 MiB/s wr, 652 op/s; 189/1734 objects degraded (10.900%) 2024-04-12T21:53:24.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:23 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:24.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:23 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:24.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:23 smithi177 ceph-mon[52869]: from='osd.1 [v2:172.21.15.123:6810/3126033631,v1:172.21.15.123:6811/3126033631]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T21:53:24.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:23 smithi177 ceph-mon[52869]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T21:53:24.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224246 10.79 MB/sec execute 364 sec latency 71.853 ms 2024-04-12T21:53:25.044 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:24 smithi123 ceph-mon[67226]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T21:53:25.044 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:24 smithi123 ceph-mon[67226]: osdmap e89: 8 total, 7 up, 8 in 2024-04-12T21:53:25.044 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:24 smithi123 ceph-mon[67226]: from='osd.1 [v2:172.21.15.123:6810/3126033631,v1:172.21.15.123:6811/3126033631]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:53:25.045 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:24 smithi123 ceph-mon[67226]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:53:25.045 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:25.045 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:25.045 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:24 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 189/1734 objects degraded (10.900%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:53:25.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:24 smithi177 ceph-mon[52869]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T21:53:25.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:24 smithi177 ceph-mon[52869]: osdmap e89: 8 total, 7 up, 8 in 2024-04-12T21:53:25.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:24 smithi177 ceph-mon[52869]: from='osd.1 [v2:172.21.15.123:6810/3126033631,v1:172.21.15.123:6811/3126033631]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:53:25.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:24 smithi177 ceph-mon[52869]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:53:25.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:25.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:25.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:24 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 189/1734 objects degraded (10.900%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:53:25.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224418 10.76 MB/sec execute 365 sec latency 605.491 ms 2024-04-12T21:53:26.059 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:25 smithi123 ceph-mon[67226]: pgmap v112: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 550 op/s; 189/1734 objects degraded (10.900%) 2024-04-12T21:53:26.059 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:25 smithi123 ceph-mon[67226]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:53:26.059 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:25 smithi123 ceph-mon[67226]: osd.1 [v2:172.21.15.123:6810/3126033631,v1:172.21.15.123:6811/3126033631] boot 2024-04-12T21:53:26.059 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:25 smithi123 ceph-mon[67226]: osdmap e90: 8 total, 8 up, 8 in 2024-04-12T21:53:26.059 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:53:26.059 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:25 smithi123 ceph-mon[67226]: osdmap e91: 8 total, 8 up, 8 in 2024-04-12T21:53:26.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:25 smithi177 ceph-mon[52869]: pgmap v112: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 550 op/s; 189/1734 objects degraded (10.900%) 2024-04-12T21:53:26.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:25 smithi177 ceph-mon[52869]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:53:26.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:25 smithi177 ceph-mon[52869]: osd.1 [v2:172.21.15.123:6810/3126033631,v1:172.21.15.123:6811/3126033631] boot 2024-04-12T21:53:26.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:25 smithi177 ceph-mon[52869]: osdmap e90: 8 total, 8 up, 8 in 2024-04-12T21:53:26.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T21:53:26.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:25 smithi177 ceph-mon[52869]: osdmap e91: 8 total, 8 up, 8 in 2024-04-12T21:53:26.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224591 10.74 MB/sec execute 366 sec latency 1081.753 ms 2024-04-12T21:53:27.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224807 10.72 MB/sec execute 367 sec latency 814.022 ms 2024-04-12T21:53:27.936 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:27 smithi123 ceph-mon[67226]: pgmap v115: 97 pgs: 1 active, 66 active+clean, 6 active+undersized, 9 active+undersized+degraded, 6 activating, 5 remapped+peering, 4 activating+degraded; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 772 op/s; 130/1734 objects degraded (7.497%) 2024-04-12T21:53:27.936 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:27 smithi123 ceph-mon[67226]: osdmap e92: 8 total, 8 up, 8 in 2024-04-12T21:53:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:27 smithi177 ceph-mon[52869]: pgmap v115: 97 pgs: 1 active, 66 active+clean, 6 active+undersized, 9 active+undersized+degraded, 6 activating, 5 remapped+peering, 4 activating+degraded; 347 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 772 op/s; 130/1734 objects degraded (7.497%) 2024-04-12T21:53:28.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:27 smithi177 ceph-mon[52869]: osdmap e92: 8 total, 8 up, 8 in 2024-04-12T21:53:28.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 225007 10.70 MB/sec execute 368 sec latency 363.514 ms 2024-04-12T21:53:28.996 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:28 smithi123 ceph-mon[67226]: osdmap e93: 8 total, 8 up, 8 in 2024-04-12T21:53:28.996 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:28 smithi123 ceph-mon[67226]: pgmap v118: 97 pgs: 1 active, 68 active+clean, 5 active+undersized, 8 active+undersized+degraded, 6 activating, 5 remapped+peering, 4 activating+degraded; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.1 MiB/s wr, 492 op/s; 115/1689 objects degraded (6.809%); 35 KiB/s, 4 objects/s recovering 2024-04-12T21:53:29.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:28 smithi177 ceph-mon[52869]: osdmap e93: 8 total, 8 up, 8 in 2024-04-12T21:53:29.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:28 smithi177 ceph-mon[52869]: pgmap v118: 97 pgs: 1 active, 68 active+clean, 5 active+undersized, 8 active+undersized+degraded, 6 activating, 5 remapped+peering, 4 activating+degraded; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.1 MiB/s wr, 492 op/s; 115/1689 objects degraded (6.809%); 35 KiB/s, 4 objects/s recovering 2024-04-12T21:53:29.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 225342 10.70 MB/sec execute 369 sec latency 799.693 ms 2024-04-12T21:53:30.019 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:29 smithi123 ceph-mon[67226]: osdmap e94: 8 total, 8 up, 8 in 2024-04-12T21:53:30.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:29 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 115/1689 objects degraded (6.809%), 12 pgs degraded (PG_DEGRADED) 2024-04-12T21:53:30.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:29 smithi177 ceph-mon[52869]: osdmap e94: 8 total, 8 up, 8 in 2024-04-12T21:53:30.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:29 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 115/1689 objects degraded (6.809%), 12 pgs degraded (PG_DEGRADED) 2024-04-12T21:53:30.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 225907 10.70 MB/sec execute 370 sec latency 262.845 ms 2024-04-12T21:53:31.039 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T21:53:31.039 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.039 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: pgmap v121: 97 pgs: 1 peering, 1 active, 81 active+clean, 6 activating, 4 remapped+peering, 4 activating+degraded; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 663 KiB/s wr, 39 op/s; 70/1671 objects degraded (4.189%); 36 KiB/s, 72 keys/s, 13 objects/s recovering 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: pgmap v122: 97 pgs: 1 peering, 1 active, 81 active+clean, 6 activating, 4 remapped+peering, 4 activating+degraded; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 90 KiB/s wr, 13 op/s; 70/1671 objects degraded (4.189%); 28 B/s, 88 keys/s, 10 objects/s recovering 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T21:53:31.040 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:30 smithi123 ceph-mon[67226]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T21:53:31.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: pgmap v121: 97 pgs: 1 peering, 1 active, 81 active+clean, 6 activating, 4 remapped+peering, 4 activating+degraded; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 663 KiB/s wr, 39 op/s; 70/1671 objects degraded (4.189%); 36 KiB/s, 72 keys/s, 13 objects/s recovering 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: pgmap v122: 97 pgs: 1 peering, 1 active, 81 active+clean, 6 activating, 4 remapped+peering, 4 activating+degraded; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 90 KiB/s wr, 13 op/s; 70/1671 objects degraded (4.189%); 28 B/s, 88 keys/s, 10 objects/s recovering 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:31.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:31.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:31.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T21:53:31.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T21:53:31.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:30 smithi177 ceph-mon[52869]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T21:53:31.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 226415 10.72 MB/sec execute 371 sec latency 41.786 ms 2024-04-12T21:53:32.272 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:32 smithi123 ceph-mon[67226]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 70/1671 objects degraded (4.189%), 4 pgs degraded) 2024-04-12T21:53:32.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:32 smithi177 ceph-mon[52869]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 70/1671 objects degraded (4.189%), 4 pgs degraded) 2024-04-12T21:53:32.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 226982 10.75 MB/sec execute 372 sec latency 26.103 ms 2024-04-12T21:53:33.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:33 smithi123 ceph-mon[67226]: pgmap v123: 97 pgs: 1 peering, 96 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 375 op/s; 982 KiB/s, 67 keys/s, 43 objects/s recovering 2024-04-12T21:53:33.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:33 smithi177 ceph-mon[52869]: pgmap v123: 97 pgs: 1 peering, 96 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 375 op/s; 982 KiB/s, 67 keys/s, 43 objects/s recovering 2024-04-12T21:53:33.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 227519 10.77 MB/sec execute 373 sec latency 31.503 ms 2024-04-12T21:53:34.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 227888 10.75 MB/sec execute 374 sec latency 42.110 ms 2024-04-12T21:53:35.360 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:35 smithi123 ceph-mon[67226]: pgmap v124: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.1 MiB/s wr, 528 op/s; 696 KiB/s, 47 keys/s, 36 objects/s recovering 2024-04-12T21:53:35.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:35 smithi177 ceph-mon[52869]: pgmap v124: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.1 MiB/s wr, 528 op/s; 696 KiB/s, 47 keys/s, 36 objects/s recovering 2024-04-12T21:53:35.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 228283 10.75 MB/sec execute 375 sec latency 50.412 ms 2024-04-12T21:53:36.466 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:53:36.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 228597 10.73 MB/sec execute 376 sec latency 65.348 ms 2024-04-12T21:53:37.275 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:53:37.275 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (5m) 14s ago 14m 24.3M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:53:37.275 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (109s) 14s ago 14m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:53:37.275 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (103s) 99s ago 14m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:53:37.275 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 14s ago 14m - - 2024-04-12T21:53:37.275 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (8m) 14s ago 8m 38.2M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:53:37.275 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (8m) 99s ago 8m 14.7M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (4m) 14s ago 15m 507M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (3m) 99s ago 14m 450M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (3m) 14s ago 16m 47.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (2m) 99s ago 13m 34.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (8m) 14s ago 8m 265M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (8m) 99s ago 8m 84.7M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (5m) 14s ago 14m 22.2M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (4m) 99s ago 13m 19.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (71s) 14s ago 13m 249M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (23s) 14s ago 12m 11.6M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (12m) 14s ago 12m 551M 1936M 17.2.0 e1d6a67b021e 459eb64d490d 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (11m) 14s ago 11m 395M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (11m) 99s ago 11m 561M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (11m) 99s ago 11m 439M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:53:37.276 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (10m) 99s ago 10m 347M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:53:37.277 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (10m) 99s ago 10m 564M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:53:37.277 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (3m) 14s ago 13m 67.0M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:53:37.450 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:37 smithi123 ceph-mon[67226]: pgmap v125: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.8 MiB/s wr, 684 op/s; 572 KiB/s, 25 objects/s recovering 2024-04-12T21:53:37.450 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:37 smithi123 ceph-mon[67226]: from='client.44185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:37.450 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:37 smithi123 ceph-mon[67226]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:37.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:37 smithi177 ceph-mon[52869]: pgmap v125: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.8 MiB/s wr, 684 op/s; 572 KiB/s, 25 objects/s recovering 2024-04-12T21:53:37.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:37 smithi177 ceph-mon[52869]: from='client.44185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:37.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:37 smithi177 ceph-mon[52869]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:37.536 INFO:teuthology.orchestra.run.smithi123.stdout: 5 228857 10.72 MB/sec execute 377 sec latency 71.128 ms 2024-04-12T21:53:37.750 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:53:37.751 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-12T21:53:37.752 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 6 2024-04-12T21:53:37.752 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:53:37.752 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "8/23 daemons upgraded", 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:53:38.168 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:53:38.169 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:53:38.236 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:38 smithi123 ceph-mon[67226]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:38.236 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:38 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/330086523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:38.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:38 smithi177 ceph-mon[52869]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:38.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:38 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/330086523' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:53:38.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 229135 10.71 MB/sec execute 378 sec latency 76.458 ms 2024-04-12T21:53:38.652 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:53:38.652 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:53:38.652 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:53:39.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:39 smithi123 ceph-mon[67226]: pgmap v126: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 618 op/s; 502 KiB/s, 22 objects/s recovering 2024-04-12T21:53:39.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:39 smithi123 ceph-mon[67226]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:39.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:39 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3383881300' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:53:39.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:39 smithi177 ceph-mon[52869]: pgmap v126: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 618 op/s; 502 KiB/s, 22 objects/s recovering 2024-04-12T21:53:39.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:39 smithi177 ceph-mon[52869]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:53:39.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:39 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3383881300' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:53:39.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 229436 10.70 MB/sec execute 379 sec latency 78.666 ms 2024-04-12T21:53:40.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 229724 10.69 MB/sec execute 380 sec latency 54.381 ms 2024-04-12T21:53:41.314 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:41.314 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:41 smithi123 ceph-mon[67226]: pgmap v127: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.8 MiB/s wr, 604 op/s; 409 KiB/s, 18 objects/s recovering 2024-04-12T21:53:41.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:41.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:41 smithi177 ceph-mon[52869]: pgmap v127: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.8 MiB/s wr, 604 op/s; 409 KiB/s, 18 objects/s recovering 2024-04-12T21:53:41.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 230048 10.70 MB/sec execute 381 sec latency 55.261 ms 2024-04-12T21:53:42.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 230443 10.72 MB/sec execute 382 sec latency 40.993 ms 2024-04-12T21:53:43.382 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:43 smithi123 ceph-mon[67226]: pgmap v128: 97 pgs: 97 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 662 op/s; 341 KiB/s, 15 objects/s recovering 2024-04-12T21:53:43.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:43 smithi177 ceph-mon[52869]: pgmap v128: 97 pgs: 97 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 662 op/s; 341 KiB/s, 15 objects/s recovering 2024-04-12T21:53:43.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 230795 10.72 MB/sec execute 383 sec latency 34.041 ms 2024-04-12T21:53:44.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 231036 10.71 MB/sec execute 384 sec latency 50.425 ms 2024-04-12T21:53:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:45 smithi123 ceph-mon[67226]: pgmap v129: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 541 op/s; 0 B/s, 2 objects/s recovering 2024-04-12T21:53:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:53:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:45 smithi177 ceph-mon[52869]: pgmap v129: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 541 op/s; 0 B/s, 2 objects/s recovering 2024-04-12T21:53:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:45.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:53:45.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 231221 10.69 MB/sec execute 385 sec latency 50.333 ms 2024-04-12T21:53:46.203 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T21:53:46.203 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:46 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T21:53:46.203 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:46 smithi123 ceph-mon[67226]: Upgrade: osd.2 is safe to restart 2024-04-12T21:53:46.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T21:53:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:46 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T21:53:46.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:46 smithi177 ceph-mon[52869]: Upgrade: osd.2 is safe to restart 2024-04-12T21:53:46.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 231427 10.66 MB/sec execute 386 sec latency 87.708 ms 2024-04-12T21:53:47.247 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:47 smithi123 ceph-mon[67226]: pgmap v130: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.3 MiB/s wr, 581 op/s 2024-04-12T21:53:47.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:47 smithi177 ceph-mon[52869]: pgmap v130: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.3 MiB/s wr, 581 op/s 2024-04-12T21:53:47.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 231638 10.65 MB/sec execute 387 sec latency 92.088 ms 2024-04-12T21:53:48.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 231903 10.64 MB/sec execute 388 sec latency 42.594 ms 2024-04-12T21:53:49.041 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:49 smithi123 ceph-mon[67226]: Upgrade: Updating osd.2 2024-04-12T21:53:49.041 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:49 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:49.041 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:49 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T21:53:49.041 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:49 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:53:49.041 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:49 smithi123 ceph-mon[67226]: Deploying daemon osd.2 on smithi123 2024-04-12T21:53:49.041 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:49 smithi123 ceph-mon[67226]: pgmap v131: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 436 op/s 2024-04-12T21:53:49.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:49 smithi177 ceph-mon[52869]: Upgrade: Updating osd.2 2024-04-12T21:53:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:49 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:53:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:49 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T21:53:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:49 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:53:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:49 smithi177 ceph-mon[52869]: Deploying daemon osd.2 on smithi123 2024-04-12T21:53:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:49 smithi177 ceph-mon[52869]: pgmap v131: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 436 op/s 2024-04-12T21:53:49.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 232306 10.64 MB/sec execute 389 sec latency 32.891 ms 2024-04-12T21:53:50.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 232742 10.63 MB/sec execute 390 sec latency 41.827 ms 2024-04-12T21:53:51.327 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:51 smithi123 ceph-mon[67226]: pgmap v132: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 453 op/s 2024-04-12T21:53:51.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:51 smithi177 ceph-mon[52869]: pgmap v132: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 453 op/s 2024-04-12T21:53:51.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 233130 10.62 MB/sec execute 391 sec latency 47.987 ms 2024-04-12T21:53:52.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 233527 10.64 MB/sec execute 392 sec latency 41.340 ms 2024-04-12T21:53:53.368 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:53 smithi123 ceph-mon[67226]: pgmap v133: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 484 op/s 2024-04-12T21:53:53.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:53 smithi177 ceph-mon[52869]: pgmap v133: 97 pgs: 97 active+clean; 334 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 484 op/s 2024-04-12T21:53:53.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 233932 10.65 MB/sec execute 393 sec latency 47.745 ms 2024-04-12T21:53:54.385 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:54 smithi123 systemd[1]: Stopping Ceph osd.2 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:53:54.537 INFO:teuthology.orchestra.run.smithi123.stdout: 5 234265 10.66 MB/sec execute 394 sec latency 48.841 ms 2024-04-12T21:53:54.890 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:54 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[41974]: 2024-04-12T21:53:54.642+0000 7fd3a8547700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:53:54.890 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:54 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[41974]: 2024-04-12T21:53:54.642+0000 7fd3a8547700 -1 osd.2 95 *** Got signal Terminated *** 2024-04-12T21:53:54.890 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:54 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[41974]: 2024-04-12T21:53:54.642+0000 7fd3a8547700 -1 osd.2 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T21:53:55.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:55 smithi123 ceph-mon[67226]: pgmap v134: 97 pgs: 97 active+clean; 335 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 475 op/s 2024-04-12T21:53:55.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:55 smithi123 ceph-mon[67226]: osd.2 marked itself down and dead 2024-04-12T21:53:55.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:55 smithi177 ceph-mon[52869]: pgmap v134: 97 pgs: 97 active+clean; 335 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 475 op/s 2024-04-12T21:53:55.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:55 smithi177 ceph-mon[52869]: osd.2 marked itself down and dead 2024-04-12T21:53:55.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 234448 10.65 MB/sec execute 395 sec latency 381.155 ms 2024-04-12T21:53:56.168 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:55 smithi123 podman[78407]: 2024-04-12 21:53:55.831027844 +0000 UTC m=+1.322884082 container died 459eb64d490da217ef7934285864d58d2f6c71acd1ae1fc53214a0d2fe0079ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, release=754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., version=8, vcs-type=git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-12T21:53:56.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:56 smithi123 ceph-mon[67226]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:53:56.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:56 smithi123 ceph-mon[67226]: osdmap e96: 8 total, 7 up, 8 in 2024-04-12T21:53:56.419 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:56 smithi123 podman[78407]: 2024-04-12 21:53:56.208840962 +0000 UTC m=+1.700697200 container cleanup 459eb64d490da217ef7934285864d58d2f6c71acd1ae1fc53214a0d2fe0079ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, name=centos-stream, io.openshift.expose-services=, version=8, architecture=x86_64, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., RELEASE=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, release=754) 2024-04-12T21:53:56.419 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:56 smithi123 bash[78407]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2 2024-04-12T21:53:56.419 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:56 smithi123 podman[78421]: 2024-04-12 21:53:56.41031106 +0000 UTC m=+0.569390151 container remove 459eb64d490da217ef7934285864d58d2f6c71acd1ae1fc53214a0d2fe0079ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-12T21:53:56.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:56 smithi177 ceph-mon[52869]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:53:56.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:56 smithi177 ceph-mon[52869]: osdmap e96: 8 total, 7 up, 8 in 2024-04-12T21:53:56.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 234525 10.62 MB/sec execute 396 sec latency 1083.671 ms 2024-04-12T21:53:56.933 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:56 smithi123 podman[78516]: 2024-04-12 21:53:56.816398249 +0000 UTC m=+0.019116924 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:53:57.184 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:57 smithi123 ceph-mon[67226]: pgmap v136: 97 pgs: 18 peering, 1 stale+active+clean, 78 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 420 op/s 2024-04-12T21:53:57.184 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:57 smithi123 ceph-mon[67226]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-12T21:53:57.184 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:57 smithi123 ceph-mon[67226]: osdmap e97: 8 total, 7 up, 8 in 2024-04-12T21:53:57.184 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:56 smithi123 podman[78516]: 2024-04-12 21:53:56.936604016 +0000 UTC m=+0.139322674 container create 0dea7183b09e5b36778ffb3724fc88199d36c25b99bc08a2a0a4fa4382c9fb1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-12T21:53:57.435 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:57 smithi123 podman[78516]: 2024-04-12 21:53:57.303289234 +0000 UTC m=+0.506007892 container init 0dea7183b09e5b36778ffb3724fc88199d36c25b99bc08a2a0a4fa4382c9fb1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T21:53:57.435 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:57 smithi123 podman[78516]: 2024-04-12 21:53:57.308145125 +0000 UTC m=+0.510863786 container start 0dea7183b09e5b36778ffb3724fc88199d36c25b99bc08a2a0a4fa4382c9fb1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-04-12T21:53:57.435 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:57 smithi123 podman[78516]: 2024-04-12 21:53:57.41608613 +0000 UTC m=+0.618804789 container attach 0dea7183b09e5b36778ffb3724fc88199d36c25b99bc08a2a0a4fa4382c9fb1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T21:53:57.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:57 smithi177 ceph-mon[52869]: pgmap v136: 97 pgs: 18 peering, 1 stale+active+clean, 78 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 420 op/s 2024-04-12T21:53:57.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:57 smithi177 ceph-mon[52869]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-12T21:53:57.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:57 smithi177 ceph-mon[52869]: osdmap e97: 8 total, 7 up, 8 in 2024-04-12T21:53:57.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 234751 10.60 MB/sec execute 397 sec latency 54.227 ms 2024-04-12T21:53:57.695 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:57 smithi123 podman[78516]: 2024-04-12 21:53:57.683480262 +0000 UTC m=+0.886198928 container died 0dea7183b09e5b36778ffb3724fc88199d36c25b99bc08a2a0a4fa4382c9fb1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_CLEAN=True, ceph=True) 2024-04-12T21:53:58.454 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:58 smithi123 ceph-mon[67226]: Health check failed: Degraded data redundancy: 19/1704 objects degraded (1.115%), 1 pg degraded (PG_DEGRADED) 2024-04-12T21:53:58.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:58 smithi177 ceph-mon[52869]: Health check failed: Degraded data redundancy: 19/1704 objects degraded (1.115%), 1 pg degraded (PG_DEGRADED) 2024-04-12T21:53:58.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 235036 10.59 MB/sec execute 398 sec latency 44.184 ms 2024-04-12T21:53:58.966 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:58 smithi123 podman[78542]: 2024-04-12 21:53:58.91290904 +0000 UTC m=+1.224294653 container remove 0dea7183b09e5b36778ffb3724fc88199d36c25b99bc08a2a0a4fa4382c9fb1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T21:53:58.966 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:58 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-12T21:53:58.966 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:58 smithi123 systemd[1]: Stopped Ceph osd.2 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:53:58.966 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:58 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.2.service: Consumed 2min 59.296s CPU time. 2024-04-12T21:53:59.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:53:59 smithi123 ceph-mon[67226]: pgmap v138: 97 pgs: 2 active+undersized, 1 active+undersized+degraded, 18 peering, 1 stale+active+clean, 75 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 491 op/s; 19/1704 objects degraded (1.115%) 2024-04-12T21:53:59.475 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:59 smithi123 systemd[1]: Starting Ceph osd.2 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:53:59.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:53:59 smithi177 ceph-mon[52869]: pgmap v138: 97 pgs: 2 active+undersized, 1 active+undersized+degraded, 18 peering, 1 stale+active+clean, 75 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 491 op/s; 19/1704 objects degraded (1.115%) 2024-04-12T21:53:59.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 235296 10.57 MB/sec execute 399 sec latency 49.330 ms 2024-04-12T21:53:59.730 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:59 smithi123 podman[78643]: 2024-04-12 21:53:59.39008438 +0000 UTC m=+0.021126214 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:53:59.730 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:59 smithi123 podman[78643]: 2024-04-12 21:53:59.515311741 +0000 UTC m=+0.146353555 container create bc473c3fef21d846842cb646e1cc12d965ac1708288eadd84628e7ad36dc7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-12T21:53:59.983 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:59 smithi123 podman[78643]: 2024-04-12 21:53:59.873457745 +0000 UTC m=+0.504499576 container init bc473c3fef21d846842cb646e1cc12d965ac1708288eadd84628e7ad36dc7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb) 2024-04-12T21:53:59.983 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:59 smithi123 podman[78643]: 2024-04-12 21:53:59.879052568 +0000 UTC m=+0.510094385 container start bc473c3fef21d846842cb646e1cc12d965ac1708288eadd84628e7ad36dc7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-b277feb, GIT_BRANCH=HEAD) 2024-04-12T21:53:59.983 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:53:59 smithi123 podman[78643]: 2024-04-12 21:53:59.973542645 +0000 UTC m=+0.604584477 container attach bc473c3fef21d846842cb646e1cc12d965ac1708288eadd84628e7ad36dc7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True) 2024-04-12T21:54:00.486 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate[78657]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T21:54:00.486 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate[78657]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T21:54:00.486 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 bash[78643]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T21:54:00.486 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 bash[78643]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 bash[78643]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 bash[78643]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 bash[78643]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 bash[78643]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 bash[78643]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate[78657]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate[78657]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate[78657]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate[78657]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T21:54:00.487 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate[78657]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-12T21:54:00.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 235599 10.56 MB/sec execute 400 sec latency 44.174 ms 2024-04-12T21:54:00.592 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:00 smithi123 podman[78643]: 2024-04-12 21:54:00.509944905 +0000 UTC m=+1.140986735 container died bc473c3fef21d846842cb646e1cc12d965ac1708288eadd84628e7ad36dc7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:54:01.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:01.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:54:01.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:01 smithi123 ceph-mon[67226]: pgmap v139: 97 pgs: 4 active+undersized, 4 active+undersized+degraded, 18 peering, 71 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 KiB/s rd, 3.9 MiB/s wr, 369 op/s; 56/1707 objects degraded (3.281%) 2024-04-12T21:54:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:54:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:01 smithi177 ceph-mon[52869]: pgmap v139: 97 pgs: 4 active+undersized, 4 active+undersized+degraded, 18 peering, 71 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.6 KiB/s rd, 3.9 MiB/s wr, 369 op/s; 56/1707 objects degraded (3.281%) 2024-04-12T21:54:01.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 235914 10.56 MB/sec execute 401 sec latency 50.085 ms 2024-04-12T21:54:01.858 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:01 smithi123 podman[78778]: 2024-04-12 21:54:01.729781839 +0000 UTC m=+1.204581950 container remove bc473c3fef21d846842cb646e1cc12d965ac1708288eadd84628e7ad36dc7077 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:54:02.110 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:01 smithi123 podman[78835]: 2024-04-12 21:54:01.882607085 +0000 UTC m=+0.024009847 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:54:02.110 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:02 smithi123 podman[78835]: 2024-04-12 21:54:02.008601744 +0000 UTC m=+0.150004501 container create 013228f4e63d19e433bdfb1e8df28a3e49d5b06c3aa7a972fe847dc4e232f648 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-12T21:54:02.373 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:02 smithi123 ceph-mon[67226]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-12T21:54:02.373 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:02 smithi123 podman[78835]: 2024-04-12 21:54:02.253430337 +0000 UTC m=+0.394833092 container init 013228f4e63d19e433bdfb1e8df28a3e49d5b06c3aa7a972fe847dc4e232f648 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:54:02.373 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:02 smithi123 podman[78835]: 2024-04-12 21:54:02.263417792 +0000 UTC m=+0.404820547 container start 013228f4e63d19e433bdfb1e8df28a3e49d5b06c3aa7a972fe847dc4e232f648 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T21:54:02.373 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:02 smithi123 ceph-osd[78854]: -- 172.21.15.123:0/895560338 <== mon.0 v2:172.21.15.123:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x565101005680 con 0x565100fcf000 2024-04-12T21:54:02.373 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:02 smithi123 bash[78835]: 013228f4e63d19e433bdfb1e8df28a3e49d5b06c3aa7a972fe847dc4e232f648 2024-04-12T21:54:02.373 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:02 smithi123 systemd[1]: Started Ceph osd.2 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:54:02.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:02 smithi177 ceph-mon[52869]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-12T21:54:02.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 236299 10.55 MB/sec execute 402 sec latency 53.382 ms 2024-04-12T21:54:02.876 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:02 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[78850]: 2024-04-12T21:54:02.849+0000 7fdb4e3887c0 -1 Falling back to public interface 2024-04-12T21:54:03.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 236732 10.55 MB/sec execute 403 sec latency 57.997 ms 2024-04-12T21:54:03.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:03 smithi123 ceph-mon[67226]: pgmap v140: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 405 op/s; 167/1620 objects degraded (10.309%) 2024-04-12T21:54:03.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:03.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:03.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:54:03.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:03 smithi177 ceph-mon[52869]: pgmap v140: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 405 op/s; 167/1620 objects degraded (10.309%) 2024-04-12T21:54:03.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:03.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:03.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:54:04.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 237193 10.57 MB/sec execute 404 sec latency 39.147 ms 2024-04-12T21:54:04.659 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:04 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 165/1614 objects degraded (10.223%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:04.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:04 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 165/1614 objects degraded (10.223%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:05.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 237603 10.58 MB/sec execute 405 sec latency 30.605 ms 2024-04-12T21:54:05.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:05 smithi123 ceph-mon[67226]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 369 op/s; 165/1614 objects degraded (10.223%) 2024-04-12T21:54:05.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:05 smithi177 ceph-mon[52869]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 369 op/s; 165/1614 objects degraded (10.223%) 2024-04-12T21:54:06.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 237974 10.59 MB/sec execute 406 sec latency 34.826 ms 2024-04-12T21:54:07.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 238266 10.58 MB/sec execute 407 sec latency 34.826 ms 2024-04-12T21:54:07.697 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:07 smithi123 ceph-mon[67226]: pgmap v142: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 522 op/s; 170/1722 objects degraded (9.872%) 2024-04-12T21:54:07.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:07 smithi177 ceph-mon[52869]: pgmap v142: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 522 op/s; 170/1722 objects degraded (9.872%) 2024-04-12T21:54:08.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 238596 10.57 MB/sec execute 408 sec latency 36.117 ms 2024-04-12T21:54:09.223 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:54:09.542 INFO:teuthology.orchestra.run.smithi123.stdout: 5 238829 10.55 MB/sec execute 409 sec latency 84.794 ms 2024-04-12T21:54:09.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:09 smithi123 ceph-mon[67226]: pgmap v143: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 452 op/s; 174/1740 objects degraded (10.000%) 2024-04-12T21:54:09.733 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:09 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[78850]: 2024-04-12T21:54:09.711+0000 7fdb4e3887c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-12T21:54:09.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:09 smithi177 ceph-mon[52869]: pgmap v143: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 452 op/s; 174/1740 objects degraded (10.000%) 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (6m) 47s ago 15m 24.3M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (2m) 47s ago 15m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (2m) 2m ago 14m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 47s ago 14m - - 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (9m) 47s ago 9m 38.2M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (9m) 2m ago 9m 14.7M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (5m) 47s ago 16m 507M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (3m) 47s ago 16m 47.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (3m) 2m ago 14m 34.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (9m) 47s ago 9m 265M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:54:10.340 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (8m) 2m ago 8m 84.7M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (5m) 47s ago 14m 22.2M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (5m) 2m ago 14m 19.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (104s) 47s ago 13m 249M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (56s) 47s ago 13m 11.6M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 starting - - - 1936M 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (12m) 47s ago 12m 395M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (12m) 2m ago 12m 561M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (11m) 2m ago 11m 439M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (11m) 2m ago 11m 347M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (10m) 2m ago 10m 564M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:54:10.341 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (4m) 47s ago 14m 67.0M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:54:10.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 239093 10.53 MB/sec execute 410 sec latency 57.994 ms 2024-04-12T21:54:10.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:10 smithi123 ceph-mon[67226]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:10.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:10 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 174/1740 objects degraded (10.000%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:10.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:10 smithi123 ceph-mon[67226]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:10.748 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:10.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:10 smithi177 ceph-mon[52869]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:10.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:10 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 174/1740 objects degraded (10.000%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:10.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:10 smithi177 ceph-mon[52869]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:10.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:11.026 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:54:11.026 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:54:11.026 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:54:11.026 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:54:11.026 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 6 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:54:11.027 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:54:11.262 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:11 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[78850]: 2024-04-12T21:54:11.107+0000 7fdb4e3887c0 -1 osd.2 95 log_to_monitors true 2024-04-12T21:54:11.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 239378 10.53 MB/sec execute 411 sec latency 65.209 ms 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:54:11.584 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "9/23 daemons upgraded", 2024-04-12T21:54:11.585 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:54:11.585 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:54:11.585 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:54:11.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:11 smithi123 ceph-mon[67226]: pgmap v144: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 456 op/s; 177/1746 objects degraded (10.137%) 2024-04-12T21:54:11.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:11 smithi123 ceph-mon[67226]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:11.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:11 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/952968698' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:11.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:11 smithi123 ceph-mon[67226]: from='osd.2 [v2:172.21.15.123:6818/1968627619,v1:172.21.15.123:6819/1968627619]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T21:54:11.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:11 smithi123 ceph-mon[67226]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T21:54:11.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:11 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:11.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:11 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:11.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:11 smithi177 ceph-mon[52869]: pgmap v144: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 456 op/s; 177/1746 objects degraded (10.137%) 2024-04-12T21:54:11.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:11 smithi177 ceph-mon[52869]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:11.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:11 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/952968698' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:11.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:11 smithi177 ceph-mon[52869]: from='osd.2 [v2:172.21.15.123:6818/1968627619,v1:172.21.15.123:6819/1968627619]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T21:54:11.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:11 smithi177 ceph-mon[52869]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T21:54:11.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:11 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:11.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:11 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:12.243 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 177/1746 objects degraded (10.137%), 20 pgs degraded 2024-04-12T21:54:12.243 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:54:12.243 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:54:12.243 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: osd.2 (root=default,host=smithi123) is down 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 177/1746 objects degraded (10.137%), 20 pgs degraded 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-04-12T21:54:12.244 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-04-12T21:54:12.245 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-04-12T21:54:12.245 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-04-12T21:54:12.245 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-04-12T21:54:12.245 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-04-12T21:54:12.245 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-04-12T21:54:12.245 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-04-12T21:54:12.245 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-04-12T21:54:12.245 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-04-12T21:54:12.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 239589 10.51 MB/sec execute 412 sec latency 125.385 ms 2024-04-12T21:54:12.563 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:12 smithi123 ceph-mon[67226]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:12.563 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:12 smithi123 ceph-mon[67226]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T21:54:12.563 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:12 smithi123 ceph-mon[67226]: osdmap e98: 8 total, 7 up, 8 in 2024-04-12T21:54:12.563 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:12 smithi123 ceph-mon[67226]: from='osd.2 [v2:172.21.15.123:6818/1968627619,v1:172.21.15.123:6819/1968627619]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:54:12.564 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:12 smithi123 ceph-mon[67226]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:54:12.564 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:12 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/523174190' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:54:12.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:12 smithi177 ceph-mon[52869]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:12.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:12 smithi177 ceph-mon[52869]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T21:54:12.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:12 smithi177 ceph-mon[52869]: osdmap e98: 8 total, 7 up, 8 in 2024-04-12T21:54:12.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:12 smithi177 ceph-mon[52869]: from='osd.2 [v2:172.21.15.123:6818/1968627619,v1:172.21.15.123:6819/1968627619]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:54:12.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:12 smithi177 ceph-mon[52869]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:54:12.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:12 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/523174190' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:54:12.826 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 21:54:12 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[78850]: 2024-04-12T21:54:12.636+0000 7fdb3b7d8700 -1 osd.2 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:54:13.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 239958 10.51 MB/sec execute 413 sec latency 61.284 ms 2024-04-12T21:54:13.863 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:13 smithi123 ceph-mon[67226]: pgmap v146: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 462 op/s; 165/1590 objects degraded (10.377%) 2024-04-12T21:54:13.863 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:13 smithi123 ceph-mon[67226]: from='osd.2 ' entity='osd.2' 2024-04-12T21:54:13.863 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:13 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:13.864 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:13 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:14.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:13 smithi177 ceph-mon[52869]: pgmap v146: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 462 op/s; 165/1590 objects degraded (10.377%) 2024-04-12T21:54:14.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:13 smithi177 ceph-mon[52869]: from='osd.2 ' entity='osd.2' 2024-04-12T21:54:14.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:13 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:14.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:13 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:14.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 240114 10.49 MB/sec execute 414 sec latency 867.482 ms 2024-04-12T21:54:14.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:14 smithi123 ceph-mon[67226]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:54:14.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:14 smithi123 ceph-mon[67226]: osd.2 [v2:172.21.15.123:6818/1968627619,v1:172.21.15.123:6819/1968627619] boot 2024-04-12T21:54:14.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:14 smithi123 ceph-mon[67226]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T21:54:14.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:14 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:54:14.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:14 smithi123 ceph-mon[67226]: osdmap e100: 8 total, 8 up, 8 in 2024-04-12T21:54:15.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:14 smithi177 ceph-mon[52869]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:54:15.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:14 smithi177 ceph-mon[52869]: osd.2 [v2:172.21.15.123:6818/1968627619,v1:172.21.15.123:6819/1968627619] boot 2024-04-12T21:54:15.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:14 smithi177 ceph-mon[52869]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T21:54:15.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:14 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T21:54:15.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:14 smithi177 ceph-mon[52869]: osdmap e100: 8 total, 8 up, 8 in 2024-04-12T21:54:15.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 240291 10.48 MB/sec execute 415 sec latency 1018.874 ms 2024-04-12T21:54:15.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:15 smithi123 ceph-mon[67226]: pgmap v148: 97 pgs: 4 peering, 4 active+undersized, 18 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 387 op/s; 134/1563 objects degraded (8.573%) 2024-04-12T21:54:15.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:15 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 134/1563 objects degraded (8.573%), 18 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:15.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:15.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:54:15.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:15 smithi123 ceph-mon[67226]: osdmap e101: 8 total, 8 up, 8 in 2024-04-12T21:54:16.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:15 smithi177 ceph-mon[52869]: pgmap v148: 97 pgs: 4 peering, 4 active+undersized, 18 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 387 op/s; 134/1563 objects degraded (8.573%) 2024-04-12T21:54:16.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:15 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 134/1563 objects degraded (8.573%), 18 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:16.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:16.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:54:16.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:15 smithi177 ceph-mon[52869]: osdmap e101: 8 total, 8 up, 8 in 2024-04-12T21:54:16.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 240558 10.48 MB/sec execute 416 sec latency 1099.238 ms 2024-04-12T21:54:17.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 240841 10.48 MB/sec execute 417 sec latency 52.193 ms 2024-04-12T21:54:17.888 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:17 smithi123 ceph-mon[67226]: pgmap v151: 97 pgs: 1 active+recovery_wait+degraded, 4 peering, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 4 active+undersized+degraded, 79 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 515 op/s; 97/1578 objects degraded (6.147%); 2.9 MiB/s, 7 objects/s recovering 2024-04-12T21:54:17.888 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:17 smithi123 ceph-mon[67226]: osdmap e102: 8 total, 8 up, 8 in 2024-04-12T21:54:18.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:17 smithi177 ceph-mon[52869]: pgmap v151: 97 pgs: 1 active+recovery_wait+degraded, 4 peering, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 4 active+undersized+degraded, 79 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 515 op/s; 97/1578 objects degraded (6.147%); 2.9 MiB/s, 7 objects/s recovering 2024-04-12T21:54:18.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:17 smithi177 ceph-mon[52869]: osdmap e102: 8 total, 8 up, 8 in 2024-04-12T21:54:18.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 241100 10.48 MB/sec execute 418 sec latency 899.165 ms 2024-04-12T21:54:18.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:18 smithi123 ceph-mon[67226]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T21:54:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:18 smithi177 ceph-mon[52869]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T21:54:19.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 241355 10.48 MB/sec execute 419 sec latency 1053.382 ms 2024-04-12T21:54:19.684 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: pgmap v154: 97 pgs: 1 active+recovery_wait+degraded, 6 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 85 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 300 op/s; 67/1584 objects degraded (4.230%); 2.1 MiB/s, 14 objects/s recovering 2024-04-12T21:54:19.684 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T21:54:19.684 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:19.684 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:19.684 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T21:54:19.685 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:19 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 67/1584 objects degraded (4.230%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: pgmap v154: 97 pgs: 1 active+recovery_wait+degraded, 6 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 85 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 300 op/s; 67/1584 objects degraded (4.230%); 2.1 MiB/s, 14 objects/s recovering 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:20.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:20.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T21:54:20.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:19 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 67/1584 objects degraded (4.230%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:20.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 241596 10.46 MB/sec execute 420 sec latency 40.316 ms 2024-04-12T21:54:20.705 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:20 smithi123 ceph-mon[67226]: pgmap v156: 97 pgs: 1 active+recovery_wait+degraded, 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 89 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.4 MiB/s wr, 368 op/s; 67/1608 objects degraded (4.167%); 2.3 MiB/s, 28 objects/s recovering 2024-04-12T21:54:20.705 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:20 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T21:54:20.705 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:20 smithi123 ceph-mon[67226]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T21:54:21.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:20 smithi177 ceph-mon[52869]: pgmap v156: 97 pgs: 1 active+recovery_wait+degraded, 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 89 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.4 MiB/s wr, 368 op/s; 67/1608 objects degraded (4.167%); 2.3 MiB/s, 28 objects/s recovering 2024-04-12T21:54:21.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:20 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T21:54:21.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:20 smithi177 ceph-mon[52869]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-12T21:54:21.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 241860 10.45 MB/sec execute 421 sec latency 63.675 ms 2024-04-12T21:54:21.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:21 smithi123 ceph-mon[67226]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/1608 objects degraded (4.167%), 4 pgs degraded) 2024-04-12T21:54:22.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:21 smithi177 ceph-mon[52869]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 67/1608 objects degraded (4.167%), 4 pgs degraded) 2024-04-12T21:54:22.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 242061 10.44 MB/sec execute 422 sec latency 62.549 ms 2024-04-12T21:54:22.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:22 smithi123 ceph-mon[67226]: pgmap v157: 97 pgs: 2 peering, 95 active+clean; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.2 MiB/s wr, 268 op/s; 0 B/s, 30 objects/s recovering 2024-04-12T21:54:23.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:22 smithi177 ceph-mon[52869]: pgmap v157: 97 pgs: 2 peering, 95 active+clean; 343 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.2 MiB/s wr, 268 op/s; 0 B/s, 30 objects/s recovering 2024-04-12T21:54:23.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 242264 10.42 MB/sec execute 423 sec latency 83.496 ms 2024-04-12T21:54:24.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 242494 10.41 MB/sec execute 424 sec latency 52.737 ms 2024-04-12T21:54:24.815 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:24 smithi123 ceph-mon[67226]: pgmap v158: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.6 MiB/s wr, 221 op/s; 116 KiB/s, 23 objects/s recovering 2024-04-12T21:54:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:24 smithi177 ceph-mon[52869]: pgmap v158: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.6 MiB/s wr, 221 op/s; 116 KiB/s, 23 objects/s recovering 2024-04-12T21:54:25.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 242720 10.40 MB/sec execute 425 sec latency 126.586 ms 2024-04-12T21:54:26.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 242971 10.38 MB/sec execute 426 sec latency 46.303 ms 2024-04-12T21:54:26.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:26 smithi123 ceph-mon[67226]: pgmap v159: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.6 KiB/s rd, 3.2 MiB/s wr, 290 op/s; 557 KiB/s, 16 objects/s recovering 2024-04-12T21:54:27.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:26 smithi177 ceph-mon[52869]: pgmap v159: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.6 KiB/s rd, 3.2 MiB/s wr, 290 op/s; 557 KiB/s, 16 objects/s recovering 2024-04-12T21:54:27.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 243295 10.38 MB/sec execute 427 sec latency 61.819 ms 2024-04-12T21:54:28.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 243634 10.37 MB/sec execute 428 sec latency 34.441 ms 2024-04-12T21:54:28.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:28 smithi123 ceph-mon[67226]: pgmap v160: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 408 op/s; 464 KiB/s, 14 objects/s recovering 2024-04-12T21:54:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:28 smithi177 ceph-mon[52869]: pgmap v160: 97 pgs: 97 active+clean; 333 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 408 op/s; 464 KiB/s, 14 objects/s recovering 2024-04-12T21:54:29.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 243970 10.38 MB/sec execute 429 sec latency 46.996 ms 2024-04-12T21:54:30.544 INFO:teuthology.orchestra.run.smithi123.stdout: 5 244374 10.38 MB/sec execute 430 sec latency 46.592 ms 2024-04-12T21:54:30.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:30 smithi123 ceph-mon[67226]: pgmap v161: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 419 op/s; 401 KiB/s, 12 objects/s recovering 2024-04-12T21:54:30.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:30.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:54:30.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:31.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:30 smithi177 ceph-mon[52869]: pgmap v161: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 419 op/s; 401 KiB/s, 12 objects/s recovering 2024-04-12T21:54:31.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:31.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:54:31.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:31.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 244690 10.39 MB/sec execute 431 sec latency 32.144 ms 2024-04-12T21:54:32.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 245026 10.39 MB/sec execute 432 sec latency 33.146 ms 2024-04-12T21:54:32.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:32 smithi123 ceph-mon[67226]: pgmap v162: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 431 op/s; 341 KiB/s, 6 objects/s recovering 2024-04-12T21:54:33.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:32 smithi177 ceph-mon[52869]: pgmap v162: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 431 op/s; 341 KiB/s, 6 objects/s recovering 2024-04-12T21:54:33.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 245383 10.38 MB/sec execute 433 sec latency 40.014 ms 2024-04-12T21:54:34.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 245700 10.37 MB/sec execute 434 sec latency 69.762 ms 2024-04-12T21:54:34.863 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:34 smithi123 ceph-mon[67226]: pgmap v163: 97 pgs: 97 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 357 op/s; 341 KiB/s, 0 objects/s recovering 2024-04-12T21:54:34.863 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:34 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T21:54:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:34 smithi177 ceph-mon[52869]: pgmap v163: 97 pgs: 97 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 357 op/s; 341 KiB/s, 0 objects/s recovering 2024-04-12T21:54:35.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:34 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T21:54:35.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 245989 10.36 MB/sec execute 435 sec latency 68.166 ms 2024-04-12T21:54:35.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:35 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T21:54:35.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:35 smithi123 ceph-mon[67226]: Upgrade: osd.3 is safe to restart 2024-04-12T21:54:36.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:35 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T21:54:36.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:35 smithi177 ceph-mon[52869]: Upgrade: osd.3 is safe to restart 2024-04-12T21:54:36.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 246293 10.35 MB/sec execute 436 sec latency 106.304 ms 2024-04-12T21:54:36.909 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:36 smithi123 ceph-mon[67226]: pgmap v164: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 404 op/s 2024-04-12T21:54:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:36 smithi177 ceph-mon[52869]: pgmap v164: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 404 op/s 2024-04-12T21:54:37.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 246759 10.35 MB/sec execute 437 sec latency 30.472 ms 2024-04-12T21:54:38.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 247014 10.34 MB/sec execute 438 sec latency 59.255 ms 2024-04-12T21:54:38.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:38 smithi123 ceph-mon[67226]: pgmap v165: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 566 op/s 2024-04-12T21:54:38.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:38 smithi123 ceph-mon[67226]: Upgrade: Updating osd.3 2024-04-12T21:54:38.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:38 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:38.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:38 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T21:54:38.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:38 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:54:38.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:38 smithi123 ceph-mon[67226]: Deploying daemon osd.3 on smithi123 2024-04-12T21:54:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:38 smithi177 ceph-mon[52869]: pgmap v165: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 566 op/s 2024-04-12T21:54:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:38 smithi177 ceph-mon[52869]: Upgrade: Updating osd.3 2024-04-12T21:54:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:38 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:38 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T21:54:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:38 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:54:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:38 smithi177 ceph-mon[52869]: Deploying daemon osd.3 on smithi123 2024-04-12T21:54:39.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 247365 10.36 MB/sec execute 439 sec latency 40.622 ms 2024-04-12T21:54:40.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 247630 10.35 MB/sec execute 440 sec latency 59.931 ms 2024-04-12T21:54:40.694 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:40 smithi123 ceph-mon[67226]: pgmap v166: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 445 op/s 2024-04-12T21:54:41.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:40 smithi177 ceph-mon[52869]: pgmap v166: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 445 op/s 2024-04-12T21:54:41.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 247945 10.34 MB/sec execute 441 sec latency 89.468 ms 2024-04-12T21:54:42.494 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:42 smithi123 systemd[1]: Stopping Ceph osd.3 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:54:42.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 248204 10.34 MB/sec execute 442 sec latency 61.377 ms 2024-04-12T21:54:42.702 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:54:42.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:42 smithi123 ceph-mon[67226]: pgmap v167: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 440 op/s 2024-04-12T21:54:42.999 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:42 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[45840]: 2024-04-12T21:54:42.769+0000 7f7d79e1a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:54:42.999 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:42 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[45840]: 2024-04-12T21:54:42.769+0000 7f7d79e1a700 -1 osd.3 104 *** Got signal Terminated *** 2024-04-12T21:54:42.999 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:42 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[45840]: 2024-04-12T21:54:42.769+0000 7f7d79e1a700 -1 osd.3 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T21:54:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:42 smithi177 ceph-mon[52869]: pgmap v167: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 440 op/s 2024-04-12T21:54:43.510 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:54:43.510 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (6m) 32s ago 15m 24.3M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:54:43.510 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (2m) 32s ago 15m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:54:43.510 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (2m) 2m ago 15m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:54:43.510 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 32s ago 15m - - 2024-04-12T21:54:43.510 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (9m) 32s ago 9m 38.5M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:54:43.510 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (9m) 2m ago 9m 14.7M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:54:43.510 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (5m) 32s ago 16m 509M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (4m) 2m ago 15m 450M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (4m) 32s ago 17m 47.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (3m) 2m ago 15m 34.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (9m) 32s ago 9m 281M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (9m) 2m ago 9m 84.7M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (6m) 32s ago 15m 23.2M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (5m) 2m ago 14m 19.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (2m) 32s ago 14m 315M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (89s) 32s ago 13m 247M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (41s) 32s ago 13m 15.0M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (13m) 32s ago 13m 418M 1936M 17.2.0 e1d6a67b021e 7bdc22b6c6c0 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (12m) 2m ago 12m 561M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (12m) 2m ago 12m 439M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:54:43.511 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (11m) 2m ago 11m 347M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:54:43.512 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (11m) 2m ago 11m 564M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:54:43.512 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (4m) 32s ago 14m 67.1M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:54:43.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 248463 10.34 MB/sec execute 443 sec latency 54.612 ms 2024-04-12T21:54:43.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:43 smithi123 ceph-mon[67226]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:43.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:43 smithi123 ceph-mon[67226]: osd.3 marked itself down and dead 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-12T21:54:43.980 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 3 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 7 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:54:43.981 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:54:44.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:43 smithi177 ceph-mon[52869]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:44.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:43 smithi177 ceph-mon[52869]: osd.3 marked itself down and dead 2024-04-12T21:54:44.408 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:54:44.408 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:54:44.408 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:54:44.408 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "9/23 daemons upgraded", 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:54:44.409 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:54:44.528 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:44 smithi123 podman[80969]: 2024-04-12 21:54:44.390318126 +0000 UTC m=+1.755794397 container died 7bdc22b6c6c01fa29170930847ed04bf9f8bd90f3bcef6896f89aeaa8b1bb939 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., distribution-scope=public, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8) 2024-04-12T21:54:44.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 248673 10.32 MB/sec execute 444 sec latency 823.053 ms 2024-04-12T21:54:44.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:44 smithi123 ceph-mon[67226]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:44.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:44 smithi123 ceph-mon[67226]: pgmap v168: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 471 op/s 2024-04-12T21:54:44.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:44 smithi123 ceph-mon[67226]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:44.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:44 smithi123 ceph-mon[67226]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:54:44.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:44 smithi123 ceph-mon[67226]: osdmap e105: 8 total, 7 up, 8 in 2024-04-12T21:54:44.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:44 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/61681050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:44.883 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down 2024-04-12T21:54:44.883 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:54:44.883 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:54:44.883 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T21:54:44.883 INFO:teuthology.orchestra.run.smithi123.stdout: osd.3 (root=default,host=smithi123) is down 2024-04-12T21:54:45.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:44 smithi177 ceph-mon[52869]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:45.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:44 smithi177 ceph-mon[52869]: pgmap v168: 97 pgs: 97 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 471 op/s 2024-04-12T21:54:45.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:44 smithi177 ceph-mon[52869]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:45.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:44 smithi177 ceph-mon[52869]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:54:45.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:44 smithi177 ceph-mon[52869]: osdmap e105: 8 total, 7 up, 8 in 2024-04-12T21:54:45.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:44 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/61681050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:54:45.293 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:45 smithi123 podman[80969]: 2024-04-12 21:54:45.107297393 +0000 UTC m=+2.472773663 container cleanup 7bdc22b6c6c01fa29170930847ed04bf9f8bd90f3bcef6896f89aeaa8b1bb939 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, distribution-scope=public, GIT_CLEAN=True, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-12T21:54:45.293 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:45 smithi123 bash[80969]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3 2024-04-12T21:54:45.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 248889 10.31 MB/sec execute 445 sec latency 1007.068 ms 2024-04-12T21:54:45.808 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:45 smithi123 ceph-mon[67226]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:45.808 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:45 smithi123 ceph-mon[67226]: osdmap e106: 8 total, 7 up, 8 in 2024-04-12T21:54:45.808 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:45 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/770395192' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:54:45.809 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:45.809 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:54:45.809 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:45 smithi123 podman[81082]: 2024-04-12 21:54:45.584614558 +0000 UTC m=+1.194299525 container remove 7bdc22b6c6c01fa29170930847ed04bf9f8bd90f3bcef6896f89aeaa8b1bb939 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, release=754, RELEASE=HEAD, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, name=centos-stream, architecture=x86_64, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux ) 2024-04-12T21:54:46.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:45 smithi177 ceph-mon[52869]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:54:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:45 smithi177 ceph-mon[52869]: osdmap e106: 8 total, 7 up, 8 in 2024-04-12T21:54:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:45 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/770395192' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:54:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:54:46.060 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:46 smithi123 podman[81229]: 2024-04-12 21:54:45.95177474 +0000 UTC m=+0.019857297 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:54:46.311 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:46 smithi123 podman[81229]: 2024-04-12 21:54:46.07734038 +0000 UTC m=+0.145422923 container create c82e21284d0a4fdd79ea75de74515f96f01ae52ed310e592f1ffc5cd2359dd9c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-12T21:54:46.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 249155 10.31 MB/sec execute 446 sec latency 69.445 ms 2024-04-12T21:54:46.563 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:46 smithi123 podman[81229]: 2024-04-12 21:54:46.476738362 +0000 UTC m=+0.544820910 container init c82e21284d0a4fdd79ea75de74515f96f01ae52ed310e592f1ffc5cd2359dd9c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=) 2024-04-12T21:54:46.563 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:46 smithi123 podman[81229]: 2024-04-12 21:54:46.485372212 +0000 UTC m=+0.553454761 container start c82e21284d0a4fdd79ea75de74515f96f01ae52ed310e592f1ffc5cd2359dd9c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-12T21:54:46.690 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:46 smithi123 podman[81229]: 2024-04-12 21:54:46.674151635 +0000 UTC m=+0.742234186 container attach c82e21284d0a4fdd79ea75de74515f96f01ae52ed310e592f1ffc5cd2359dd9c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, ceph=True) 2024-04-12T21:54:46.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:46 smithi123 ceph-mon[67226]: pgmap v171: 97 pgs: 10 peering, 12 stale+active+clean, 75 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s wr, 252 op/s 2024-04-12T21:54:46.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:46 smithi123 ceph-mon[67226]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-12T21:54:47.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:46 smithi177 ceph-mon[52869]: pgmap v171: 97 pgs: 10 peering, 12 stale+active+clean, 75 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 2.6 MiB/s wr, 252 op/s 2024-04-12T21:54:47.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:46 smithi177 ceph-mon[52869]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-12T21:54:47.069 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:46 smithi123 podman[81229]: 2024-04-12 21:54:46.861625633 +0000 UTC m=+0.929708185 container died c82e21284d0a4fdd79ea75de74515f96f01ae52ed310e592f1ffc5cd2359dd9c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, ceph=True) 2024-04-12T21:54:47.545 INFO:teuthology.orchestra.run.smithi123.stdout: 5 249497 10.30 MB/sec execute 447 sec latency 70.531 ms 2024-04-12T21:54:47.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:47 smithi123 ceph-mon[67226]: Health check failed: Degraded data redundancy: 185/1674 objects degraded (11.051%), 21 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:48.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:47 smithi177 ceph-mon[52869]: Health check failed: Degraded data redundancy: 185/1674 objects degraded (11.051%), 21 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:48.115 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:47 smithi123 podman[81254]: 2024-04-12 21:54:47.95788303 +0000 UTC m=+1.090544227 container remove c82e21284d0a4fdd79ea75de74515f96f01ae52ed310e592f1ffc5cd2359dd9c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:54:48.117 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:47 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-12T21:54:48.118 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:47 smithi123 systemd[1]: Stopped Ceph osd.3 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:54:48.118 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:47 smithi123 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.3.service: Consumed 5min 2.314s CPU time. 2024-04-12T21:54:48.378 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:48 smithi123 systemd[1]: Starting Ceph osd.3 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:54:48.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 249805 10.29 MB/sec execute 448 sec latency 53.965 ms 2024-04-12T21:54:48.630 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:48 smithi123 podman[81359]: 2024-04-12 21:54:48.436990725 +0000 UTC m=+0.025525655 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:54:48.630 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:48 smithi123 podman[81359]: 2024-04-12 21:54:48.559739718 +0000 UTC m=+0.148274633 container create 1da1b44977c3c95619a2b8a56d8b686d7a7f3e5ac9ef9673fda2646c176664df (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5) 2024-04-12T21:54:48.882 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:48 smithi123 ceph-mon[67226]: pgmap v172: 97 pgs: 10 active+undersized, 21 active+undersized+degraded, 10 peering, 56 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.2 MiB/s wr, 470 op/s; 185/1674 objects degraded (11.051%) 2024-04-12T21:54:48.883 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:48 smithi123 podman[81359]: 2024-04-12 21:54:48.829406534 +0000 UTC m=+0.417941449 container init 1da1b44977c3c95619a2b8a56d8b686d7a7f3e5ac9ef9673fda2646c176664df (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, ceph=True) 2024-04-12T21:54:48.883 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:48 smithi123 podman[81359]: 2024-04-12 21:54:48.83420702 +0000 UTC m=+0.422741941 container start 1da1b44977c3c95619a2b8a56d8b686d7a7f3e5ac9ef9673fda2646c176664df (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T21:54:49.034 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:48 smithi177 ceph-mon[52869]: pgmap v172: 97 pgs: 10 active+undersized, 21 active+undersized+degraded, 10 peering, 56 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.2 MiB/s wr, 470 op/s; 185/1674 objects degraded (11.051%) 2024-04-12T21:54:49.133 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:48 smithi123 podman[81359]: 2024-04-12 21:54:48.960767787 +0000 UTC m=+0.549302711 container attach 1da1b44977c3c95619a2b8a56d8b686d7a7f3e5ac9ef9673fda2646c176664df (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS) 2024-04-12T21:54:49.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 250156 10.28 MB/sec execute 449 sec latency 33.476 ms 2024-04-12T21:54:49.650 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 bash[81359]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T21:54:49.650 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 bash[81359]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 bash[81359]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 bash[81359]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 bash[81359]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 bash[81359]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 bash[81359]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate[81373]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate[81373]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate[81373]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate[81373]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate[81373]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate[81373]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate[81373]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T21:54:49.651 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:49 smithi123 podman[81359]: 2024-04-12 21:54:49.458764789 +0000 UTC m=+1.047299712 container died 1da1b44977c3c95619a2b8a56d8b686d7a7f3e5ac9ef9673fda2646c176664df (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T21:54:49.901 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:49 smithi123 ceph-mon[67226]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-12T21:54:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:49 smithi177 ceph-mon[52869]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-12T21:54:50.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 250429 10.28 MB/sec execute 450 sec latency 60.386 ms 2024-04-12T21:54:50.690 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:50 smithi123 podman[81509]: 2024-04-12 21:54:50.436060262 +0000 UTC m=+0.971548104 container remove 1da1b44977c3c95619a2b8a56d8b686d7a7f3e5ac9ef9673fda2646c176664df (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-12T21:54:50.690 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:50 smithi123 podman[81570]: 2024-04-12 21:54:50.568737807 +0000 UTC m=+0.020089161 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:54:50.942 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:50 smithi123 ceph-mon[67226]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.8 MiB/s wr, 402 op/s; 218/1677 objects degraded (12.999%) 2024-04-12T21:54:50.942 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:50 smithi123 podman[81570]: 2024-04-12 21:54:50.690276458 +0000 UTC m=+0.141627806 container create 7ca563e272ac33b3e59a55a216c99ee82eb4876489f1d24bcedef2655b2259a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-b277feb) 2024-04-12T21:54:51.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:50 smithi177 ceph-mon[52869]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.8 MiB/s wr, 402 op/s; 218/1677 objects degraded (12.999%) 2024-04-12T21:54:51.463 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:51 smithi123 podman[81570]: 2024-04-12 21:54:51.391731372 +0000 UTC m=+0.843082717 container init 7ca563e272ac33b3e59a55a216c99ee82eb4876489f1d24bcedef2655b2259a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240409) 2024-04-12T21:54:51.463 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:51 smithi123 podman[81570]: 2024-04-12 21:54:51.396004995 +0000 UTC m=+0.847356337 container start 7ca563e272ac33b3e59a55a216c99ee82eb4876489f1d24bcedef2655b2259a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:54:51.463 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:51 smithi123 ceph-osd[81588]: -- 172.21.15.123:0/4056216445 <== mon.0 v2:172.21.15.123:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f8fdcf5680 con 0x55f8fdcbec00 2024-04-12T21:54:51.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 250815 10.28 MB/sec execute 451 sec latency 51.149 ms 2024-04-12T21:54:51.725 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:51 smithi123 bash[81570]: 7ca563e272ac33b3e59a55a216c99ee82eb4876489f1d24bcedef2655b2259a7 2024-04-12T21:54:51.725 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:51 smithi123 systemd[1]: Started Ceph osd.3 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:54:51.985 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:54:51 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T21:54:51.966+0000 7f2b64cf87c0 -1 Falling back to public interface 2024-04-12T21:54:52.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 251158 10.28 MB/sec execute 452 sec latency 62.149 ms 2024-04-12T21:54:52.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:52 smithi123 ceph-mon[67226]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 508 op/s; 217/1698 objects degraded (12.780%) 2024-04-12T21:54:52.746 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:52.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:52.747 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:54:53.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:52 smithi177 ceph-mon[52869]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 508 op/s; 217/1698 objects degraded (12.780%) 2024-04-12T21:54:53.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:53.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:54:53.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:54:53.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 251493 10.28 MB/sec execute 453 sec latency 74.842 ms 2024-04-12T21:54:53.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:53 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 213/1701 objects degraded (12.522%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:54.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:53 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 213/1701 objects degraded (12.522%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T21:54:54.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 251821 10.28 MB/sec execute 454 sec latency 49.356 ms 2024-04-12T21:54:54.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:54 smithi123 ceph-mon[67226]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 213/1701 objects degraded (12.522%) 2024-04-12T21:54:55.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:54 smithi177 ceph-mon[52869]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 213/1701 objects degraded (12.522%) 2024-04-12T21:54:55.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 252044 10.27 MB/sec execute 455 sec latency 54.955 ms 2024-04-12T21:54:56.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 252327 10.27 MB/sec execute 456 sec latency 50.652 ms 2024-04-12T21:54:56.819 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:56 smithi123 ceph-mon[67226]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 412 op/s; 210/1710 objects degraded (12.281%) 2024-04-12T21:54:57.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:56 smithi177 ceph-mon[52869]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 412 op/s; 210/1710 objects degraded (12.281%) 2024-04-12T21:54:57.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 252576 10.26 MB/sec execute 457 sec latency 94.319 ms 2024-04-12T21:54:58.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 252834 10.25 MB/sec execute 458 sec latency 56.857 ms 2024-04-12T21:54:58.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:58 smithi123 ceph-mon[67226]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 571 op/s; 211/1716 objects degraded (12.296%) 2024-04-12T21:54:59.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:58 smithi177 ceph-mon[52869]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 571 op/s; 211/1716 objects degraded (12.296%) 2024-04-12T21:54:59.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 253118 10.24 MB/sec execute 459 sec latency 67.105 ms 2024-04-12T21:54:59.906 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:54:59 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 211/1716 objects degraded (12.296%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:00.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:54:59 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 211/1716 objects degraded (12.296%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:00.411 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:55:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T21:55:00.157+0000 7f2b64cf87c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-12T21:55:00.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 253483 10.23 MB/sec execute 460 sec latency 46.100 ms 2024-04-12T21:55:00.914 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:00 smithi123 ceph-mon[67226]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 424 op/s; 210/1680 objects degraded (12.500%) 2024-04-12T21:55:00.914 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:00.914 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:55:00.914 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:00.914 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:01.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:00 smithi177 ceph-mon[52869]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 424 op/s; 210/1680 objects degraded (12.500%) 2024-04-12T21:55:01.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:01.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:55:01.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:01.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:01.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 253873 10.23 MB/sec execute 461 sec latency 54.500 ms 2024-04-12T21:55:02.183 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:55:01 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T21:55:01.938+0000 7f2b64cf87c0 -1 osd.3 104 log_to_monitors true 2024-04-12T21:55:02.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 254230 10.23 MB/sec execute 462 sec latency 42.291 ms 2024-04-12T21:55:02.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:02 smithi123 ceph-mon[67226]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.9 MiB/s wr, 503 op/s; 221/1731 objects degraded (12.767%) 2024-04-12T21:55:02.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:02 smithi123 ceph-mon[67226]: from='osd.3 [v2:172.21.15.123:6826/3280882620,v1:172.21.15.123:6827/3280882620]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T21:55:02.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:02 smithi123 ceph-mon[67226]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T21:55:02.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:02 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:02.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:02 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:03.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:02 smithi177 ceph-mon[52869]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.9 MiB/s wr, 503 op/s; 221/1731 objects degraded (12.767%) 2024-04-12T21:55:03.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:02 smithi177 ceph-mon[52869]: from='osd.3 [v2:172.21.15.123:6826/3280882620,v1:172.21.15.123:6827/3280882620]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T21:55:03.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:02 smithi177 ceph-mon[52869]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T21:55:03.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:02 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:03.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:02 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:03.487 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 21:55:03 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T21:55:03.284+0000 7f2b52148700 -1 osd.3 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:55:03.546 INFO:teuthology.orchestra.run.smithi123.stdout: 5 254565 10.24 MB/sec execute 463 sec latency 150.991 ms 2024-04-12T21:55:03.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:03 smithi123 ceph-mon[67226]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T21:55:03.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:03 smithi123 ceph-mon[67226]: from='osd.3 [v2:172.21.15.123:6826/3280882620,v1:172.21.15.123:6827/3280882620]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:55:03.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:03 smithi123 ceph-mon[67226]: osdmap e107: 8 total, 7 up, 8 in 2024-04-12T21:55:03.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:03 smithi123 ceph-mon[67226]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:55:03.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:03 smithi123 ceph-mon[67226]: from='osd.3 ' entity='osd.3' 2024-04-12T21:55:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:03 smithi177 ceph-mon[52869]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T21:55:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:03 smithi177 ceph-mon[52869]: from='osd.3 [v2:172.21.15.123:6826/3280882620,v1:172.21.15.123:6827/3280882620]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:55:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:03 smithi177 ceph-mon[52869]: osdmap e107: 8 total, 7 up, 8 in 2024-04-12T21:55:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:03 smithi177 ceph-mon[52869]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-04-12T21:55:04.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:03 smithi177 ceph-mon[52869]: from='osd.3 ' entity='osd.3' 2024-04-12T21:55:04.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 254863 10.23 MB/sec execute 464 sec latency 262.836 ms 2024-04-12T21:55:04.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:04 smithi123 ceph-mon[67226]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 451 op/s; 217/1698 objects degraded (12.780%) 2024-04-12T21:55:04.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:04 smithi123 ceph-mon[67226]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:55:04.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:04 smithi123 ceph-mon[67226]: osd.3 [v2:172.21.15.123:6826/3280882620,v1:172.21.15.123:6827/3280882620] boot 2024-04-12T21:55:04.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:04 smithi123 ceph-mon[67226]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T21:55:04.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:55:04.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:04 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 217/1698 objects degraded (12.780%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:04.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:04 smithi123 ceph-mon[67226]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T21:55:05.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:04 smithi177 ceph-mon[52869]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 451 op/s; 217/1698 objects degraded (12.780%) 2024-04-12T21:55:05.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:04 smithi177 ceph-mon[52869]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:55:05.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:04 smithi177 ceph-mon[52869]: osd.3 [v2:172.21.15.123:6826/3280882620,v1:172.21.15.123:6827/3280882620] boot 2024-04-12T21:55:05.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:04 smithi177 ceph-mon[52869]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T21:55:05.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T21:55:05.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:04 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 217/1698 objects degraded (12.780%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:05.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:04 smithi177 ceph-mon[52869]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T21:55:05.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 255017 10.22 MB/sec execute 465 sec latency 1262.868 ms 2024-04-12T21:55:06.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 255275 10.21 MB/sec execute 466 sec latency 1687.429 ms 2024-04-12T21:55:06.649 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:06 smithi123 ceph-mon[67226]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 380 op/s; 224/1734 objects degraded (12.918%) 2024-04-12T21:55:06.649 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:06 smithi123 ceph-mon[67226]: osdmap e110: 8 total, 8 up, 8 in 2024-04-12T21:55:07.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:06 smithi177 ceph-mon[52869]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 380 op/s; 224/1734 objects degraded (12.918%) 2024-04-12T21:55:07.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:06 smithi177 ceph-mon[52869]: osdmap e110: 8 total, 8 up, 8 in 2024-04-12T21:55:07.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 255510 10.21 MB/sec execute 467 sec latency 689.622 ms 2024-04-12T21:55:07.944 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:07 smithi123 ceph-mon[67226]: osdmap e111: 8 total, 8 up, 8 in 2024-04-12T21:55:08.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:07 smithi177 ceph-mon[52869]: osdmap e111: 8 total, 8 up, 8 in 2024-04-12T21:55:08.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 255779 10.20 MB/sec execute 468 sec latency 927.015 ms 2024-04-12T21:55:08.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: pgmap v187: 97 pgs: 1 peering, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 6 active+undersized, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 68 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 432 op/s; 220/1662 objects degraded (13.237%); 2.3 MiB/s, 12 objects/s recovering 2024-04-12T21:55:08.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: osdmap e112: 8 total, 8 up, 8 in 2024-04-12T21:55:08.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:08.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:08.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:55:08.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:55:08.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:08.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:08.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:55:08.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:08.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:08.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:08.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:08 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T21:55:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: pgmap v187: 97 pgs: 1 peering, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 6 active+undersized, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 68 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 432 op/s; 220/1662 objects degraded (13.237%); 2.3 MiB/s, 12 objects/s recovering 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: osdmap e112: 8 total, 8 up, 8 in 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:09.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:08 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T21:55:09.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 256089 10.19 MB/sec execute 469 sec latency 1018.534 ms 2024-04-12T21:55:09.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:09 smithi123 ceph-mon[67226]: pgmap v189: 97 pgs: 2 peering, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 2 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 81 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 396 op/s; 165/1623 objects degraded (10.166%); 2.7 MiB/s, 28 objects/s recovering 2024-04-12T21:55:09.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:09 smithi123 ceph-mon[67226]: pgmap v190: 97 pgs: 2 peering, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 2 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 81 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 493 op/s; 165/1623 objects degraded (10.166%); 3.4 MiB/s, 35 objects/s recovering 2024-04-12T21:55:09.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:09 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T21:55:09.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:09 smithi123 ceph-mon[67226]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-12T21:55:09.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:09 smithi123 ceph-mon[67226]: osdmap e113: 8 total, 8 up, 8 in 2024-04-12T21:55:09.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:09 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 165/1623 objects degraded (10.166%), 12 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:09.995 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:09 smithi123 ceph-mon[67226]: osdmap e114: 8 total, 8 up, 8 in 2024-04-12T21:55:10.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:09 smithi177 ceph-mon[52869]: pgmap v189: 97 pgs: 2 peering, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 2 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 81 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 396 op/s; 165/1623 objects degraded (10.166%); 2.7 MiB/s, 28 objects/s recovering 2024-04-12T21:55:10.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:09 smithi177 ceph-mon[52869]: pgmap v190: 97 pgs: 2 peering, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 2 active+undersized, 6 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 81 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 493 op/s; 165/1623 objects degraded (10.166%); 3.4 MiB/s, 35 objects/s recovering 2024-04-12T21:55:10.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:09 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T21:55:10.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:09 smithi177 ceph-mon[52869]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-12T21:55:10.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:09 smithi177 ceph-mon[52869]: osdmap e113: 8 total, 8 up, 8 in 2024-04-12T21:55:10.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:09 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 165/1623 objects degraded (10.166%), 12 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:10.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:09 smithi177 ceph-mon[52869]: osdmap e114: 8 total, 8 up, 8 in 2024-04-12T21:55:10.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 256313 10.18 MB/sec execute 470 sec latency 890.096 ms 2024-04-12T21:55:11.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 256536 10.17 MB/sec execute 471 sec latency 1023.884 ms 2024-04-12T21:55:11.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:11 smithi123 ceph-mon[67226]: pgmap v193: 97 pgs: 2 peering, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 302 op/s; 146/1623 objects degraded (8.996%); 403 KiB/s, 23 objects/s recovering 2024-04-12T21:55:11.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:11 smithi123 ceph-mon[67226]: osdmap e115: 8 total, 8 up, 8 in 2024-04-12T21:55:12.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:11 smithi177 ceph-mon[52869]: pgmap v193: 97 pgs: 2 peering, 1 active+recovering+degraded, 1 activating+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 302 op/s; 146/1623 objects degraded (8.996%); 403 KiB/s, 23 objects/s recovering 2024-04-12T21:55:12.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:11 smithi177 ceph-mon[52869]: osdmap e115: 8 total, 8 up, 8 in 2024-04-12T21:55:12.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 256860 10.17 MB/sec execute 472 sec latency 35.222 ms 2024-04-12T21:55:13.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 257230 10.17 MB/sec execute 473 sec latency 59.009 ms 2024-04-12T21:55:13.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:13 smithi123 ceph-mon[67226]: pgmap v195: 97 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 327 op/s; 42/1653 objects degraded (2.541%); 1.6 MiB/s, 33 objects/s recovering 2024-04-12T21:55:14.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:13 smithi177 ceph-mon[52869]: pgmap v195: 97 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 327 op/s; 42/1653 objects degraded (2.541%); 1.6 MiB/s, 33 objects/s recovering 2024-04-12T21:55:14.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 257525 10.16 MB/sec execute 474 sec latency 58.878 ms 2024-04-12T21:55:14.719 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:14 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 42/1653 objects degraded (2.541%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:15.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:14 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 42/1653 objects degraded (2.541%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:15.376 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:55:15.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 257852 10.16 MB/sec execute 475 sec latency 46.943 ms 2024-04-12T21:55:15.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:15 smithi123 ceph-mon[67226]: pgmap v196: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.9 MiB/s wr, 422 op/s; 1.3 MiB/s, 33 objects/s recovering 2024-04-12T21:55:15.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:15 smithi123 ceph-mon[67226]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1653 objects degraded (2.541%), 2 pgs degraded) 2024-04-12T21:55:15.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:15.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:55:16.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:15 smithi177 ceph-mon[52869]: pgmap v196: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.9 MiB/s wr, 422 op/s; 1.3 MiB/s, 33 objects/s recovering 2024-04-12T21:55:16.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:15 smithi177 ceph-mon[52869]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1653 objects degraded (2.541%), 2 pgs degraded) 2024-04-12T21:55:16.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:16.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:55:16.430 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:55:16.430 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (7m) 16s ago 16m 23.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:55:16.430 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (3m) 16s ago 16m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:55:16.430 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (3m) 3m ago 15m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 16s ago 16m - - 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (10m) 16s ago 10m 38.2M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (10m) 3m ago 10m 14.7M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (6m) 16s ago 17m 510M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (5m) 3m ago 15m 450M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (4m) 16s ago 17m 49.7M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (4m) 3m ago 15m 34.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (10m) 16s ago 10m 283M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (10m) 3m ago 10m 84.7M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (6m) 16s ago 15m 19.3M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (6m) 3m ago 15m 19.8M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (2m) 16s ago 14m 332M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:55:16.431 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (2m) 16s ago 14m 283M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:55:16.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (74s) 16s ago 14m 220M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:55:16.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (25s) 16s ago 13m 13.5M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:55:16.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (13m) 3m ago 13m 561M 2704M 17.2.0 e1d6a67b021e a252cc2ecf65 2024-04-12T21:55:16.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (12m) 3m ago 12m 439M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:55:16.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (12m) 3m ago 12m 347M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:55:16.432 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (11m) 3m ago 11m 564M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:55:16.432 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (5m) 16s ago 15m 69.2M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:55:16.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 258273 10.17 MB/sec execute 476 sec latency 28.861 ms 2024-04-12T21:55:16.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:16 smithi123 ceph-mon[67226]: from='client.44245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:16.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:16 smithi123 ceph-mon[67226]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:17.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:16 smithi177 ceph-mon[52869]: from='client.44245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:17.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:16 smithi177 ceph-mon[52869]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:55:17.063 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:55:17.064 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:55:17.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 258628 10.17 MB/sec execute 477 sec latency 43.995 ms 2024-04-12T21:55:17.593 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:55:17.593 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:55:17.593 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:55:17.593 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:55:17.593 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:55:17.594 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:55:17.594 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:55:17.594 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:55:17.594 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:55:17.594 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "10/23 daemons upgraded", 2024-04-12T21:55:17.594 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:55:17.594 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:55:17.594 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:55:17.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:17 smithi123 ceph-mon[67226]: pgmap v197: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 572 op/s; 1.1 MiB/s, 27 objects/s recovering 2024-04-12T21:55:17.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:17 smithi123 ceph-mon[67226]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:17.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:17 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/2714073618' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:18.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:17 smithi177 ceph-mon[52869]: pgmap v197: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 572 op/s; 1.1 MiB/s, 27 objects/s recovering 2024-04-12T21:55:18.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:17 smithi177 ceph-mon[52869]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:18.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:17 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/2714073618' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:18.232 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:55:18.232 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:55:18.232 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:55:18.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 258978 10.17 MB/sec execute 478 sec latency 43.063 ms 2024-04-12T21:55:18.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:18 smithi123 ceph-mon[67226]: from='client.44257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:18.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:18 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/4127000499' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:55:19.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:18 smithi177 ceph-mon[52869]: from='client.44257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:19.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:18 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/4127000499' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:55:19.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 259432 10.17 MB/sec execute 479 sec latency 41.094 ms 2024-04-12T21:55:19.777 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:19 smithi123 ceph-mon[67226]: pgmap v198: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 505 op/s; 851 KiB/s, 21 objects/s recovering 2024-04-12T21:55:20.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:19 smithi177 ceph-mon[52869]: pgmap v198: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 505 op/s; 851 KiB/s, 21 objects/s recovering 2024-04-12T21:55:20.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 259835 10.16 MB/sec execute 480 sec latency 33.748 ms 2024-04-12T21:55:21.307 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:21 smithi123 ceph-mon[67226]: pgmap v199: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 435 op/s; 718 KiB/s, 18 objects/s recovering 2024-04-12T21:55:21.307 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:21.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:21 smithi177 ceph-mon[52869]: pgmap v199: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 435 op/s; 718 KiB/s, 18 objects/s recovering 2024-04-12T21:55:21.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:21.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 260082 10.16 MB/sec execute 481 sec latency 65.299 ms 2024-04-12T21:55:22.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 260464 10.16 MB/sec execute 482 sec latency 53.972 ms 2024-04-12T21:55:23.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:23 smithi123 ceph-mon[67226]: pgmap v200: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.9 MiB/s wr, 584 op/s; 628 KiB/s, 16 objects/s recovering 2024-04-12T21:55:23.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:23 smithi177 ceph-mon[52869]: pgmap v200: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.9 MiB/s wr, 584 op/s; 628 KiB/s, 16 objects/s recovering 2024-04-12T21:55:23.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 260781 10.16 MB/sec execute 483 sec latency 75.954 ms 2024-04-12T21:55:24.133 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T21:55:24.133 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:24 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T21:55:24.133 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:24 smithi123 ceph-mon[67226]: Upgrade: osd.4 is safe to restart 2024-04-12T21:55:24.234 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T21:55:24.234 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:24 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T21:55:24.234 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:24 smithi177 ceph-mon[52869]: Upgrade: osd.4 is safe to restart 2024-04-12T21:55:24.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 261037 10.15 MB/sec execute 484 sec latency 60.840 ms 2024-04-12T21:55:25.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:25 smithi123 ceph-mon[67226]: pgmap v201: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.0 MiB/s wr, 532 op/s; 117 KiB/s, 5 objects/s recovering 2024-04-12T21:55:25.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:25 smithi177 ceph-mon[52869]: pgmap v201: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.0 MiB/s wr, 532 op/s; 117 KiB/s, 5 objects/s recovering 2024-04-12T21:55:25.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 261308 10.14 MB/sec execute 485 sec latency 62.087 ms 2024-04-12T21:55:26.443 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:26 smithi123 ceph-mon[67226]: Upgrade: Updating osd.4 2024-04-12T21:55:26.443 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:26.443 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T21:55:26.443 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:55:26.443 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:26 smithi123 ceph-mon[67226]: Deploying daemon osd.4 on smithi177 2024-04-12T21:55:26.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:26 smithi177 ceph-mon[52869]: Upgrade: Updating osd.4 2024-04-12T21:55:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T21:55:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:55:26.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:26 smithi177 ceph-mon[52869]: Deploying daemon osd.4 on smithi177 2024-04-12T21:55:26.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 261564 10.14 MB/sec execute 486 sec latency 59.895 ms 2024-04-12T21:55:27.489 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:27 smithi123 ceph-mon[67226]: pgmap v202: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.5 MiB/s wr, 579 op/s 2024-04-12T21:55:27.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:27 smithi177 ceph-mon[52869]: pgmap v202: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.5 MiB/s wr, 579 op/s 2024-04-12T21:55:27.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 261768 10.13 MB/sec execute 487 sec latency 72.209 ms 2024-04-12T21:55:28.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262034 10.13 MB/sec execute 488 sec latency 54.858 ms 2024-04-12T21:55:29.188 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:28 smithi177 systemd[1]: Stopping Ceph osd.4 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:55:29.313 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:29 smithi123 ceph-mon[67226]: pgmap v203: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-04-12T21:55:29.313 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:29 smithi123 ceph-mon[67226]: osd.4 marked itself down and dead 2024-04-12T21:55:29.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:29 smithi177 ceph-mon[52869]: pgmap v203: 97 pgs: 97 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-04-12T21:55:29.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:29 smithi177 ceph-mon[52869]: osd.4 marked itself down and dead 2024-04-12T21:55:29.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[31036]: 2024-04-12T21:55:29.235+0000 7f385f44f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:55:29.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[31036]: 2024-04-12T21:55:29.235+0000 7f385f44f700 -1 osd.4 115 *** Got signal Terminated *** 2024-04-12T21:55:29.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[31036]: 2024-04-12T21:55:29.235+0000 7f385f44f700 -1 osd.4 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T21:55:29.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262234 10.12 MB/sec execute 489 sec latency 209.553 ms 2024-04-12T21:55:30.136 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:29 smithi177 podman[57358]: 2024-04-12 21:55:29.825116255 +0000 UTC m=+0.734763480 container died a252cc2ecf658e1597d196305527104ff385c594aa12cb6572132215ea230b97 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vcs-type=git, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-12T21:55:30.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:55:30.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: osdmap e116: 8 total, 7 up, 8 in 2024-04-12T21:55:30.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 4]}]: dispatch 2024-04-12T21:55:30.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.15", "id": [0, 4]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 1]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1e", "id": [0, 4]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [3, 4]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [7, 1]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:30.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:55:30.518 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:55:30.518 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: osdmap e116: 8 total, 7 up, 8 in 2024-04-12T21:55:30.518 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 4]}]: dispatch 2024-04-12T21:55:30.518 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-04-12T21:55:30.518 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]: dispatch 2024-04-12T21:55:30.518 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]: dispatch 2024-04-12T21:55:30.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.15", "id": [0, 4]}]: dispatch 2024-04-12T21:55:30.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 1]}]: dispatch 2024-04-12T21:55:30.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1e", "id": [0, 4]}]: dispatch 2024-04-12T21:55:30.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [3, 4]}]: dispatch 2024-04-12T21:55:30.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [7, 1]}]: dispatch 2024-04-12T21:55:30.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-04-12T21:55:30.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:30.519 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:55:30.519 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:30 smithi177 podman[57358]: 2024-04-12 21:55:30.516803343 +0000 UTC m=+1.426450576 container cleanup a252cc2ecf658e1597d196305527104ff385c594aa12cb6572132215ea230b97 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, maintainer=Guillaume Abrioux , name=centos-stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-12T21:55:30.519 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:30 smithi177 bash[57358]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4 2024-04-12T21:55:30.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262342 10.11 MB/sec execute 490 sec latency 1012.127 ms 2024-04-12T21:55:31.123 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:30 smithi177 podman[57384]: 2024-04-12 21:55:30.785097251 +0000 UTC m=+0.955828242 container remove a252cc2ecf658e1597d196305527104ff385c594aa12cb6572132215ea230b97 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-type=git, RELEASE=HEAD, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, release=754, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8) 2024-04-12T21:55:31.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 4]}]': finished 2024-04-12T21:55:31.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-04-12T21:55:31.378 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]': finished 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.15", "id": [0, 4]}]': finished 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 1]}]': finished 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1e", "id": [0, 4]}]': finished 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [3, 4]}]': finished 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [7, 1]}]': finished 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: osdmap e117: 8 total, 7 up, 8 in 2024-04-12T21:55:31.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:31 smithi123 ceph-mon[67226]: pgmap v205: 97 pgs: 14 stale+active+clean, 83 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-04-12T21:55:31.407 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 4]}]': finished 2024-04-12T21:55:31.407 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-04-12T21:55:31.407 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]': finished 2024-04-12T21:55:31.407 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 2]}]': finished 2024-04-12T21:55:31.407 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.15", "id": [0, 4]}]': finished 2024-04-12T21:55:31.408 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 1]}]': finished 2024-04-12T21:55:31.408 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1e", "id": [0, 4]}]': finished 2024-04-12T21:55:31.408 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [3, 4]}]': finished 2024-04-12T21:55:31.408 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [7, 1]}]': finished 2024-04-12T21:55:31.408 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-04-12T21:55:31.408 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: osdmap e117: 8 total, 7 up, 8 in 2024-04-12T21:55:31.408 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:31 smithi177 ceph-mon[52869]: pgmap v205: 97 pgs: 14 stale+active+clean, 83 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-04-12T21:55:31.408 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:31 smithi177 podman[57475]: 2024-04-12 21:55:31.024834657 +0000 UTC m=+0.019450124 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:55:31.408 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:31 smithi177 podman[57475]: 2024-04-12 21:55:31.152733355 +0000 UTC m=+0.147348810 container create c7241bfe629e744e48491a38b91f161593502452ea995dce7e2eea7b03b9f366 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T21:55:31.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262517 10.09 MB/sec execute 491 sec latency 1404.203 ms 2024-04-12T21:55:31.738 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:31 smithi177 podman[57475]: 2024-04-12 21:55:31.405931525 +0000 UTC m=+0.400546981 container init c7241bfe629e744e48491a38b91f161593502452ea995dce7e2eea7b03b9f366 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb) 2024-04-12T21:55:31.738 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:31 smithi177 podman[57475]: 2024-04-12 21:55:31.409672276 +0000 UTC m=+0.404287739 container start c7241bfe629e744e48491a38b91f161593502452ea995dce7e2eea7b03b9f366 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb) 2024-04-12T21:55:31.738 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:31 smithi177 podman[57475]: 2024-04-12 21:55:31.531101948 +0000 UTC m=+0.525717421 container attach c7241bfe629e744e48491a38b91f161593502452ea995dce7e2eea7b03b9f366 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-12T21:55:32.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:31 smithi177 podman[57475]: 2024-04-12 21:55:31.737443711 +0000 UTC m=+0.732059167 container died c7241bfe629e744e48491a38b91f161593502452ea995dce7e2eea7b03b9f366 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-12T21:55:32.435 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:32 smithi123 ceph-mon[67226]: osdmap e118: 8 total, 7 up, 8 in 2024-04-12T21:55:32.439 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:32 smithi177 ceph-mon[52869]: osdmap e118: 8 total, 7 up, 8 in 2024-04-12T21:55:32.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262647 10.08 MB/sec execute 492 sec latency 2336.821 ms 2024-04-12T21:55:32.704 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:32 smithi177 podman[57502]: 2024-04-12 21:55:32.438144833 +0000 UTC m=+0.694240448 container remove c7241bfe629e744e48491a38b91f161593502452ea995dce7e2eea7b03b9f366 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-12T21:55:32.704 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:32 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-12T21:55:32.705 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:32 smithi177 systemd[1]: Stopped Ceph osd.4 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:55:32.705 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:32 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.4.service: Consumed 5min 4.334s CPU time. 2024-04-12T21:55:33.030 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:32 smithi177 systemd[1]: Starting Ceph osd.4 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:55:33.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:32 smithi177 podman[57596]: 2024-04-12 21:55:32.849804563 +0000 UTC m=+0.020880110 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:55:33.386 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-mon[52869]: osdmap e119: 8 total, 7 up, 8 in 2024-04-12T21:55:33.386 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-mon[52869]: 2.1a starting backfill to osd.1 from (0'0,0'0] MAX to 105'9941 2024-04-12T21:55:33.386 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-mon[52869]: 2.5 starting backfill to osd.2 from (0'0,0'0] MAX to 97'8246 2024-04-12T21:55:33.386 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-mon[52869]: 2.b starting backfill to osd.2 from (0'0,0'0] MAX to 115'9596 2024-04-12T21:55:33.386 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-mon[52869]: pgmap v209: 97 pgs: 3 remapped+peering, 6 active+undersized, 19 active+undersized+degraded, 2 peering, 4 stale+active+clean, 63 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.3 MiB/s wr, 345 op/s; 184/1626 objects degraded (11.316%) 2024-04-12T21:55:33.386 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 podman[57596]: 2024-04-12 21:55:33.123361349 +0000 UTC m=+0.294436866 container create 9a6ffb20fd6a0d26c3fe625b35e759e15e97588987d072c32100dd6174de4f68 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-12T21:55:33.461 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:33 smithi123 ceph-mon[67226]: osdmap e119: 8 total, 7 up, 8 in 2024-04-12T21:55:33.461 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:33 smithi123 ceph-mon[67226]: 2.1a starting backfill to osd.1 from (0'0,0'0] MAX to 105'9941 2024-04-12T21:55:33.461 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:33 smithi123 ceph-mon[67226]: 2.5 starting backfill to osd.2 from (0'0,0'0] MAX to 97'8246 2024-04-12T21:55:33.461 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:33 smithi123 ceph-mon[67226]: 2.b starting backfill to osd.2 from (0'0,0'0] MAX to 115'9596 2024-04-12T21:55:33.462 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:33 smithi123 ceph-mon[67226]: pgmap v209: 97 pgs: 3 remapped+peering, 6 active+undersized, 19 active+undersized+degraded, 2 peering, 4 stale+active+clean, 63 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.3 MiB/s wr, 345 op/s; 184/1626 objects degraded (11.316%) 2024-04-12T21:55:33.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262996 10.08 MB/sec execute 493 sec latency 54.417 ms 2024-04-12T21:55:33.781 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 podman[57596]: 2024-04-12 21:55:33.384723145 +0000 UTC m=+0.555798674 container init 9a6ffb20fd6a0d26c3fe625b35e759e15e97588987d072c32100dd6174de4f68 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T21:55:33.781 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 podman[57596]: 2024-04-12 21:55:33.389276745 +0000 UTC m=+0.560352268 container start 9a6ffb20fd6a0d26c3fe625b35e759e15e97588987d072c32100dd6174de4f68 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_CLEAN=True) 2024-04-12T21:55:33.781 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 podman[57596]: 2024-04-12 21:55:33.48485364 +0000 UTC m=+0.655929167 container attach 9a6ffb20fd6a0d26c3fe625b35e759e15e97588987d072c32100dd6174de4f68 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-12T21:55:34.273 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate[57611]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate[57611]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 bash[57596]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 bash[57596]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 bash[57596]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 bash[57596]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 bash[57596]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 bash[57596]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 bash[57596]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate[57611]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate[57611]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate[57611]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-12T21:55:34.274 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate[57611]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T21:55:34.275 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate[57611]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-12T21:55:34.275 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:33 smithi177 podman[57596]: 2024-04-12 21:55:33.96451494 +0000 UTC m=+1.135590462 container died 9a6ffb20fd6a0d26c3fe625b35e759e15e97588987d072c32100dd6174de4f68 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-12T21:55:34.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:34 smithi123 ceph-mon[67226]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T21:55:34.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:34 smithi123 ceph-mon[67226]: Health check failed: Degraded data redundancy: 184/1626 objects degraded (11.316%), 19 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:34.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:34 smithi123 ceph-mon[67226]: osdmap e120: 8 total, 7 up, 8 in 2024-04-12T21:55:34.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:34 smithi177 ceph-mon[52869]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T21:55:34.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:34 smithi177 ceph-mon[52869]: Health check failed: Degraded data redundancy: 184/1626 objects degraded (11.316%), 19 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:34.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:34 smithi177 ceph-mon[52869]: osdmap e120: 8 total, 7 up, 8 in 2024-04-12T21:55:34.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 263304 10.07 MB/sec execute 494 sec latency 61.736 ms 2024-04-12T21:55:35.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:34 smithi177 podman[57728]: 2024-04-12 21:55:34.606347396 +0000 UTC m=+0.635900096 container remove 9a6ffb20fd6a0d26c3fe625b35e759e15e97588987d072c32100dd6174de4f68 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0) 2024-04-12T21:55:35.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:34 smithi177 podman[57788]: 2024-04-12 21:55:34.739158817 +0000 UTC m=+0.021261207 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:55:35.313 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:35 smithi177 podman[57788]: 2024-04-12 21:55:35.057368399 +0000 UTC m=+0.339470829 container create 7dafde6ecf2c0a9191e4244d05b04cde026c882709b48f13de44ec7461effa47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T21:55:35.496 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:35 smithi123 ceph-mon[67226]: pgmap v211: 97 pgs: 4 active+clean+remapped, 3 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 2 peering, 52 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 400 op/s; 228/1611 objects degraded (14.153%); 26/1611 objects misplaced (1.614%) 2024-04-12T21:55:35.496 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:35 smithi123 ceph-mon[67226]: osdmap e121: 8 total, 7 up, 8 in 2024-04-12T21:55:35.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 263604 10.06 MB/sec execute 495 sec latency 64.225 ms 2024-04-12T21:55:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:35 smithi177 ceph-mon[52869]: pgmap v211: 97 pgs: 4 active+clean+remapped, 3 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 2 peering, 52 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 400 op/s; 228/1611 objects degraded (14.153%); 26/1611 objects misplaced (1.614%) 2024-04-12T21:55:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:35 smithi177 ceph-mon[52869]: osdmap e121: 8 total, 7 up, 8 in 2024-04-12T21:55:35.781 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:35 smithi177 podman[57788]: 2024-04-12 21:55:35.311742462 +0000 UTC m=+0.593844860 container init 7dafde6ecf2c0a9191e4244d05b04cde026c882709b48f13de44ec7461effa47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-12T21:55:35.781 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:35 smithi177 podman[57788]: 2024-04-12 21:55:35.316356654 +0000 UTC m=+0.598459044 container start 7dafde6ecf2c0a9191e4244d05b04cde026c882709b48f13de44ec7461effa47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240409, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:55:35.781 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:35 smithi177 ceph-osd[57806]: -- 172.21.15.177:0/2433249854 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5593b302b680 con 0x5593b2ff5000 2024-04-12T21:55:35.781 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:35 smithi177 bash[57788]: 7dafde6ecf2c0a9191e4244d05b04cde026c882709b48f13de44ec7461effa47 2024-04-12T21:55:35.781 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:35 smithi177 systemd[1]: Started Ceph osd.4 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:55:36.281 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:35 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T21:55:35.863+0000 7f7f997b67c0 -1 Falling back to public interface 2024-04-12T21:55:36.545 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:36 smithi123 ceph-mon[67226]: osdmap e122: 8 total, 7 up, 8 in 2024-04-12T21:55:36.545 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:36.545 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:36.545 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:55:36.547 INFO:teuthology.orchestra.run.smithi123.stdout: 5 263828 10.05 MB/sec execute 496 sec latency 113.779 ms 2024-04-12T21:55:36.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:36 smithi177 ceph-mon[52869]: osdmap e122: 8 total, 7 up, 8 in 2024-04-12T21:55:36.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:36 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:36.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:36 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:36.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:36 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:55:37.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 264103 10.04 MB/sec execute 497 sec latency 59.572 ms 2024-04-12T21:55:37.559 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:37 smithi123 ceph-mon[67226]: pgmap v214: 97 pgs: 6 active+clean+remapped, 3 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 52 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.3 MiB/s wr, 439 op/s; 225/1599 objects degraded (14.071%); 35/1599 objects misplaced (2.189%); 2.0 MiB/s, 1 keys/s, 0 objects/s recovering 2024-04-12T21:55:37.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:37 smithi177 ceph-mon[52869]: pgmap v214: 97 pgs: 6 active+clean+remapped, 3 remapped+peering, 12 active+undersized, 24 active+undersized+degraded, 52 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.3 MiB/s wr, 439 op/s; 225/1599 objects degraded (14.071%); 35/1599 objects misplaced (2.189%); 2.0 MiB/s, 1 keys/s, 0 objects/s recovering 2024-04-12T21:55:38.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 264374 10.05 MB/sec execute 498 sec latency 44.272 ms 2024-04-12T21:55:38.587 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:38 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 235/1632 objects degraded (14.400%), 24 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:38.587 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:38 smithi123 ceph-mon[67226]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T21:55:38.708 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:38 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 235/1632 objects degraded (14.400%), 24 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:38.708 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:38 smithi177 ceph-mon[52869]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T21:55:39.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 264750 10.06 MB/sec execute 499 sec latency 44.593 ms 2024-04-12T21:55:39.598 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:39 smithi123 ceph-mon[67226]: pgmap v215: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 333 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.1 MiB/s wr, 345 op/s; 235/1632 objects degraded (14.400%); 56/1632 objects misplaced (3.431%); 4.1 MiB/s, 4 keys/s, 114 objects/s recovering 2024-04-12T21:55:39.846 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:39 smithi177 ceph-mon[52869]: pgmap v215: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 333 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.1 MiB/s wr, 345 op/s; 235/1632 objects degraded (14.400%); 56/1632 objects misplaced (3.431%); 4.1 MiB/s, 4 keys/s, 114 objects/s recovering 2024-04-12T21:55:40.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 265165 10.06 MB/sec execute 500 sec latency 33.397 ms 2024-04-12T21:55:41.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:41.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:41.379 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:41 smithi123 ceph-mon[67226]: pgmap v216: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 330 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.8 MiB/s wr, 302 op/s; 239/1632 objects degraded (14.645%); 50/1632 objects misplaced (3.064%); 3.5 MiB/s, 4 keys/s, 98 objects/s recovering 2024-04-12T21:55:41.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:41.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:41.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:41 smithi177 ceph-mon[52869]: pgmap v216: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 330 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.8 MiB/s wr, 302 op/s; 239/1632 objects degraded (14.645%); 50/1632 objects misplaced (3.064%); 3.5 MiB/s, 4 keys/s, 98 objects/s recovering 2024-04-12T21:55:41.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 265478 10.05 MB/sec execute 501 sec latency 42.085 ms 2024-04-12T21:55:42.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 265756 10.05 MB/sec execute 502 sec latency 62.049 ms 2024-04-12T21:55:42.633 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:42.633 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:42.647 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:42.647 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:43.392 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:43 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T21:55:43.167+0000 7f7f997b67c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-12T21:55:43.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 266053 10.04 MB/sec execute 503 sec latency 45.430 ms 2024-04-12T21:55:43.673 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:43 smithi123 ceph-mon[67226]: pgmap v217: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 406 op/s; 246/1704 objects degraded (14.437%); 50/1704 objects misplaced (2.934%); 3.1 MiB/s, 3 keys/s, 86 objects/s recovering 2024-04-12T21:55:43.697 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:43 smithi177 ceph-mon[52869]: pgmap v217: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 406 op/s; 246/1704 objects degraded (14.437%); 50/1704 objects misplaced (2.934%); 3.1 MiB/s, 3 keys/s, 86 objects/s recovering 2024-04-12T21:55:44.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 266363 10.03 MB/sec execute 504 sec latency 52.206 ms 2024-04-12T21:55:44.689 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:44 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 243/1725 objects degraded (14.087%), 24 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:45.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:44 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 243/1725 objects degraded (14.087%), 24 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:45.031 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:44 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T21:55:44.805+0000 7f7f997b67c0 -1 osd.4 115 log_to_monitors true 2024-04-12T21:55:45.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 266694 10.03 MB/sec execute 505 sec latency 60.870 ms 2024-04-12T21:55:45.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:45 smithi123 ceph-mon[67226]: pgmap v218: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.9 MiB/s wr, 531 op/s; 243/1725 objects degraded (14.087%); 53/1725 objects misplaced (3.072%); 2.5 MiB/s, 2 keys/s, 68 objects/s recovering 2024-04-12T21:55:45.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:45 smithi123 ceph-mon[67226]: from='osd.4 [v2:172.21.15.177:6800/662211074,v1:172.21.15.177:6801/662211074]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T21:55:45.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:45 smithi123 ceph-mon[67226]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T21:55:45.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:45.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:55:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:45 smithi177 ceph-mon[52869]: pgmap v218: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.9 MiB/s wr, 531 op/s; 243/1725 objects degraded (14.087%); 53/1725 objects misplaced (3.072%); 2.5 MiB/s, 2 keys/s, 68 objects/s recovering 2024-04-12T21:55:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:45 smithi177 ceph-mon[52869]: from='osd.4 [v2:172.21.15.177:6800/662211074,v1:172.21.15.177:6801/662211074]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T21:55:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:45 smithi177 ceph-mon[52869]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T21:55:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:46.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:55:46.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T21:55:46.102+0000 7f7f86c06700 -1 osd.4 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:55:46.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 266913 10.02 MB/sec execute 506 sec latency 197.121 ms 2024-04-12T21:55:46.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: osdmap e123: 8 total, 7 up, 8 in 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='osd.4 [v2:172.21.15.177:6800/662211074,v1:172.21.15.177:6801/662211074]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: pgmap v220: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 4.1 MiB/s wr, 392 op/s; 243/1725 objects degraded (14.087%); 53/1725 objects misplaced (3.072%); 1.8 MiB/s, 2 keys/s, 72 objects/s recovering 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: pgmap v221: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.4 MiB/s wr, 457 op/s; 243/1725 objects degraded (14.087%); 53/1725 objects misplaced (3.072%) 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:46.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='osd.4 ' entity='osd.4' 2024-04-12T21:55:46.704 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:55:46.704 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:46.704 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:46.704 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:46.704 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T21:55:47.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T21:55:47.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: osdmap e123: 8 total, 7 up, 8 in 2024-04-12T21:55:47.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='osd.4 [v2:172.21.15.177:6800/662211074,v1:172.21.15.177:6801/662211074]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:55:47.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:55:47.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:47.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:47.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: pgmap v220: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 4.1 MiB/s wr, 392 op/s; 243/1725 objects degraded (14.087%); 53/1725 objects misplaced (3.072%); 1.8 MiB/s, 2 keys/s, 72 objects/s recovering 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: pgmap v221: 97 pgs: 10 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.4 MiB/s wr, 457 op/s; 243/1725 objects degraded (14.087%); 53/1725 objects misplaced (3.072%) 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='osd.4 ' entity='osd.4' 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:47.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T21:55:47.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 267209 10.01 MB/sec execute 507 sec latency 431.952 ms 2024-04-12T21:55:48.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:47 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T21:55:48.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:47 smithi123 ceph-mon[67226]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-12T21:55:48.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:47 smithi123 ceph-mon[67226]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:55:48.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:47 smithi123 ceph-mon[67226]: osd.4 [v2:172.21.15.177:6800/662211074,v1:172.21.15.177:6801/662211074] boot 2024-04-12T21:55:48.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:47 smithi123 ceph-mon[67226]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T21:55:48.061 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:47 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:55:48.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:47 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T21:55:48.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:47 smithi177 ceph-mon[52869]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-12T21:55:48.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:47 smithi177 ceph-mon[52869]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:55:48.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:47 smithi177 ceph-mon[52869]: osd.4 [v2:172.21.15.177:6800/662211074,v1:172.21.15.177:6801/662211074] boot 2024-04-12T21:55:48.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:47 smithi177 ceph-mon[52869]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T21:55:48.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:47 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T21:55:48.548 INFO:teuthology.orchestra.run.smithi123.stdout: 5 267272 10.00 MB/sec execute 508 sec latency 1431.983 ms 2024-04-12T21:55:48.621 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:55:48.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:48 smithi123 ceph-mon[67226]: pgmap v223: 97 pgs: 4 remapped+peering, 6 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 524 op/s; 233/1659 objects degraded (14.045%); 31/1659 objects misplaced (1.869%) 2024-04-12T21:55:48.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:48 smithi123 ceph-mon[67226]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T21:55:48.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:48 smithi123 ceph-mon[67226]: 2.15 starting backfill to osd.4 from (0'0,0'0] MAX to 123'7787 2024-04-12T21:55:48.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:48 smithi123 ceph-mon[67226]: 2.0 starting backfill to osd.4 from (0'0,0'0] MAX to 122'10990 2024-04-12T21:55:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:48 smithi177 ceph-mon[52869]: pgmap v223: 97 pgs: 4 remapped+peering, 6 active+clean+remapped, 12 active+undersized, 24 active+undersized+degraded, 51 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 524 op/s; 233/1659 objects degraded (14.045%); 31/1659 objects misplaced (1.869%) 2024-04-12T21:55:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:48 smithi177 ceph-mon[52869]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T21:55:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:48 smithi177 ceph-mon[52869]: 2.15 starting backfill to osd.4 from (0'0,0'0] MAX to 123'7787 2024-04-12T21:55:49.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:48 smithi177 ceph-mon[52869]: 2.0 starting backfill to osd.4 from (0'0,0'0] MAX to 122'10990 2024-04-12T21:55:49.388 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (7m) 49s ago 16m 23.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (4m) 49s ago 16m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (3m) 9s ago 16m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 49s ago 16m - - 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (11m) 49s ago 11m 38.2M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (11m) 9s ago 11m 15.4M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (6m) 49s ago 18m 510M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (5m) 9s ago 16m 452M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (5m) 49s ago 18m 49.7M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (4m) 9s ago 16m 41.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (10m) 49s ago 10m 283M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (10m) 9s ago 10m 87.3M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 49s ago 16m 19.3M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (6m) 9s ago 16m 17.3M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:55:49.389 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (3m) 49s ago 15m 332M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:55:49.390 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (2m) 49s ago 15m 283M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:55:49.390 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (107s) 49s ago 14m 220M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:55:49.390 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (58s) 49s ago 14m 13.5M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:55:49.390 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (14s) 9s ago 13m 15.1M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T21:55:49.390 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (13m) 9s ago 13m 641M 2704M 17.2.0 e1d6a67b021e e55ce008c318 2024-04-12T21:55:49.390 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (12m) 9s ago 12m 553M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:55:49.390 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (12m) 9s ago 12m 616M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:55:49.390 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (5m) 49s ago 15m 69.2M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:55:49.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 267346 9.98 MB/sec execute 509 sec latency 2046.728 ms 2024-04-12T21:55:49.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:49 smithi123 ceph-mon[67226]: 2.1e starting backfill to osd.4 from (0'0,0'0] MAX to 72'7461 2024-04-12T21:55:49.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:49 smithi123 ceph-mon[67226]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:49.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:49 smithi123 ceph-mon[67226]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T21:55:49.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:49 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 233/1659 objects degraded (14.045%), 24 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-12T21:55:50.061 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 5 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 9 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:55:50.062 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:55:50.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:49 smithi177 ceph-mon[52869]: 2.1e starting backfill to osd.4 from (0'0,0'0] MAX to 72'7461 2024-04-12T21:55:50.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:49 smithi177 ceph-mon[52869]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:50.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:49 smithi177 ceph-mon[52869]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T21:55:50.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:49 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 233/1659 objects degraded (14.045%), 24 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:50.483 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:55:50.483 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "11/23 daemons upgraded", 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:55:50.484 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:55:50.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 267609 9.97 MB/sec execute 510 sec latency 342.179 ms 2024-04-12T21:55:50.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:50 smithi123 ceph-mon[67226]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:50.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:50 smithi123 ceph-mon[67226]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:50.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:50 smithi123 ceph-mon[67226]: pgmap v226: 97 pgs: 2 peering, 7 remapped+peering, 2 active+clean+remapped, 9 active+undersized, 2 active+remapped+backfill_wait, 19 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.3 MiB/s wr, 313 op/s; 184/1608 objects degraded (11.443%); 18/1608 objects misplaced (1.119%); 38 KiB/s, 55 keys/s, 3 objects/s recovering 2024-04-12T21:55:50.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:50 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3153491182' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:50.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:50 smithi123 ceph-mon[67226]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T21:55:51.036 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 184/1608 objects degraded (11.443%), 19 pgs degraded 2024-04-12T21:55:51.036 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:55:51.036 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:55:51.036 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 184/1608 objects degraded (11.443%), 19 pgs degraded 2024-04-12T21:55:51.036 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-04-12T21:55:51.036 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-04-12T21:55:51.036 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-04-12T21:55:51.037 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-04-12T21:55:51.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:50 smithi177 ceph-mon[52869]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:51.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:50 smithi177 ceph-mon[52869]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:51.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:50 smithi177 ceph-mon[52869]: pgmap v226: 97 pgs: 2 peering, 7 remapped+peering, 2 active+clean+remapped, 9 active+undersized, 2 active+remapped+backfill_wait, 19 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.3 MiB/s wr, 313 op/s; 184/1608 objects degraded (11.443%); 18/1608 objects misplaced (1.119%); 38 KiB/s, 55 keys/s, 3 objects/s recovering 2024-04-12T21:55:51.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:50 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3153491182' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:55:51.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:50 smithi177 ceph-mon[52869]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T21:55:51.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 267933 9.98 MB/sec execute 511 sec latency 936.725 ms 2024-04-12T21:55:52.013 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:51 smithi123 ceph-mon[67226]: from='client.44281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:52.013 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:51 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/882642410' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:55:52.013 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:51 smithi123 ceph-mon[67226]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T21:55:52.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:51 smithi177 ceph-mon[52869]: from='client.44281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:55:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:51 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/882642410' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:55:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:51 smithi177 ceph-mon[52869]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T21:55:52.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 268297 9.98 MB/sec execute 512 sec latency 983.710 ms 2024-04-12T21:55:53.034 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:52 smithi123 ceph-mon[67226]: pgmap v229: 97 pgs: 3 active+recovery_wait+degraded, 4 peering, 4 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 78 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 456 op/s; 113/1566 objects degraded (7.216%); 10/1566 objects misplaced (0.639%); 2.3 MiB/s, 61 keys/s, 52 objects/s recovering 2024-04-12T21:55:53.034 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:52 smithi123 ceph-mon[67226]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T21:55:53.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:52 smithi177 ceph-mon[52869]: pgmap v229: 97 pgs: 3 active+recovery_wait+degraded, 4 peering, 4 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 78 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 456 op/s; 113/1566 objects degraded (7.216%); 10/1566 objects misplaced (0.639%); 2.3 MiB/s, 61 keys/s, 52 objects/s recovering 2024-04-12T21:55:53.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:52 smithi177 ceph-mon[52869]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T21:55:53.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 268578 9.98 MB/sec execute 513 sec latency 46.961 ms 2024-04-12T21:55:54.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 268837 9.97 MB/sec execute 514 sec latency 707.845 ms 2024-04-12T21:55:54.817 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:54 smithi123 ceph-mon[67226]: pgmap v231: 97 pgs: 3 active+recovery_wait+degraded, 2 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 308 op/s; 113/1551 objects degraded (7.286%); 11/1551 objects misplaced (0.709%); 2.0 MiB/s, 4 keys/s, 45 objects/s recovering 2024-04-12T21:55:54.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:54 smithi123 ceph-mon[67226]: osdmap e130: 8 total, 8 up, 8 in 2024-04-12T21:55:54.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:54 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 113/1551 objects degraded (7.286%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:54.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:54 smithi123 ceph-mon[67226]: osdmap e131: 8 total, 8 up, 8 in 2024-04-12T21:55:55.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:54 smithi177 ceph-mon[52869]: pgmap v231: 97 pgs: 3 active+recovery_wait+degraded, 2 peering, 1 remapped+peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 82 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 308 op/s; 113/1551 objects degraded (7.286%); 11/1551 objects misplaced (0.709%); 2.0 MiB/s, 4 keys/s, 45 objects/s recovering 2024-04-12T21:55:55.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:54 smithi177 ceph-mon[52869]: osdmap e130: 8 total, 8 up, 8 in 2024-04-12T21:55:55.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:54 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 113/1551 objects degraded (7.286%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T21:55:55.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:54 smithi177 ceph-mon[52869]: osdmap e131: 8 total, 8 up, 8 in 2024-04-12T21:55:55.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 269063 9.96 MB/sec execute 515 sec latency 885.743 ms 2024-04-12T21:55:56.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 269307 9.96 MB/sec execute 516 sec latency 1015.638 ms 2024-04-12T21:55:56.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:56 smithi123 ceph-mon[67226]: osdmap e132: 8 total, 8 up, 8 in 2024-04-12T21:55:56.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:56 smithi123 ceph-mon[67226]: pgmap v235: 97 pgs: 3 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 85 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s wr, 278 op/s; 113/1590 objects degraded (7.107%); 11/1590 objects misplaced (0.692%); 137 KiB/s, 169 objects/s recovering 2024-04-12T21:55:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:56 smithi177 ceph-mon[52869]: osdmap e132: 8 total, 8 up, 8 in 2024-04-12T21:55:57.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:56 smithi177 ceph-mon[52869]: pgmap v235: 97 pgs: 3 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 85 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s wr, 278 op/s; 113/1590 objects degraded (7.107%); 11/1590 objects misplaced (0.692%); 137 KiB/s, 169 objects/s recovering 2024-04-12T21:55:57.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 269582 9.96 MB/sec execute 517 sec latency 714.508 ms 2024-04-12T21:55:57.724 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:57 smithi123 ceph-mon[67226]: osdmap e133: 8 total, 8 up, 8 in 2024-04-12T21:55:58.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:57 smithi177 ceph-mon[52869]: osdmap e133: 8 total, 8 up, 8 in 2024-04-12T21:55:58.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 269933 9.96 MB/sec execute 518 sec latency 824.790 ms 2024-04-12T21:55:58.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:58 smithi123 ceph-mon[67226]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T21:55:58.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:58 smithi123 ceph-mon[67226]: pgmap v238: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 470 op/s; 5/1659 objects degraded (0.301%); 11/1659 objects misplaced (0.663%); 29 KiB/s, 193 objects/s recovering 2024-04-12T21:55:59.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:58 smithi177 ceph-mon[52869]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T21:55:59.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:58 smithi177 ceph-mon[52869]: pgmap v238: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 92 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 470 op/s; 5/1659 objects degraded (0.301%); 11/1659 objects misplaced (0.663%); 29 KiB/s, 193 objects/s recovering 2024-04-12T21:55:59.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 270321 9.95 MB/sec execute 519 sec latency 56.052 ms 2024-04-12T21:55:59.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:55:59 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 5/1659 objects degraded (0.301%), 1 pg degraded (PG_DEGRADED) 2024-04-12T21:56:00.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:55:59 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 5/1659 objects degraded (0.301%), 1 pg degraded (PG_DEGRADED) 2024-04-12T21:56:00.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 270576 9.94 MB/sec execute 520 sec latency 36.919 ms 2024-04-12T21:56:00.785 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:00 smithi123 ceph-mon[67226]: pgmap v239: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.5 MiB/s wr, 244 op/s; 5/1692 objects degraded (0.296%); 5/1692 objects misplaced (0.296%); 1.6 MiB/s, 0 keys/s, 150 objects/s recovering 2024-04-12T21:56:00.785 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:00.785 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:56:01.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:00 smithi177 ceph-mon[52869]: pgmap v239: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.5 MiB/s wr, 244 op/s; 5/1692 objects degraded (0.296%); 5/1692 objects misplaced (0.296%); 1.6 MiB/s, 0 keys/s, 150 objects/s recovering 2024-04-12T21:56:01.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:01.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:56:01.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 270885 9.94 MB/sec execute 521 sec latency 74.650 ms 2024-04-12T21:56:01.799 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T21:56:01.965 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T21:56:02.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 271213 9.94 MB/sec execute 522 sec latency 70.065 ms 2024-04-12T21:56:02.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:02 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T21:56:02.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:02 smithi123 ceph-mon[67226]: Upgrade: osd.5 is safe to restart 2024-04-12T21:56:02.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:02 smithi123 ceph-mon[67226]: pgmap v240: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.8 MiB/s wr, 400 op/s; 2.6 MiB/s, 3 keys/s, 20 objects/s recovering 2024-04-12T21:56:03.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:02 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T21:56:03.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:02 smithi177 ceph-mon[52869]: Upgrade: osd.5 is safe to restart 2024-04-12T21:56:03.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:02 smithi177 ceph-mon[52869]: pgmap v240: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.8 MiB/s wr, 400 op/s; 2.6 MiB/s, 3 keys/s, 20 objects/s recovering 2024-04-12T21:56:03.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 271579 9.95 MB/sec execute 523 sec latency 41.414 ms 2024-04-12T21:56:03.836 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:03 smithi123 ceph-mon[67226]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1692 objects degraded (0.296%), 1 pg degraded) 2024-04-12T21:56:04.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:03 smithi177 ceph-mon[52869]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1692 objects degraded (0.296%), 1 pg degraded) 2024-04-12T21:56:04.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 271879 9.94 MB/sec execute 524 sec latency 55.513 ms 2024-04-12T21:56:05.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:04 smithi123 ceph-mon[67226]: pgmap v241: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.8 MiB/s wr, 492 op/s; 2.0 MiB/s, 2 keys/s, 15 objects/s recovering 2024-04-12T21:56:05.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:04 smithi123 ceph-mon[67226]: Upgrade: Updating osd.5 2024-04-12T21:56:05.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:05.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T21:56:05.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:56:05.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:04 smithi123 ceph-mon[67226]: Deploying daemon osd.5 on smithi177 2024-04-12T21:56:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:04 smithi177 ceph-mon[52869]: pgmap v241: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.8 MiB/s wr, 492 op/s; 2.0 MiB/s, 2 keys/s, 15 objects/s recovering 2024-04-12T21:56:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:04 smithi177 ceph-mon[52869]: Upgrade: Updating osd.5 2024-04-12T21:56:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T21:56:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:56:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:04 smithi177 ceph-mon[52869]: Deploying daemon osd.5 on smithi177 2024-04-12T21:56:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 272185 9.93 MB/sec execute 525 sec latency 70.856 ms 2024-04-12T21:56:06.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 272433 9.92 MB/sec execute 526 sec latency 40.334 ms 2024-04-12T21:56:06.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:06 smithi123 ceph-mon[67226]: pgmap v242: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.7 MiB/s wr, 371 op/s; 1.8 MiB/s, 2 keys/s, 5 objects/s recovering 2024-04-12T21:56:07.257 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:06 smithi177 ceph-mon[52869]: pgmap v242: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.7 MiB/s wr, 371 op/s; 1.8 MiB/s, 2 keys/s, 5 objects/s recovering 2024-04-12T21:56:07.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 272756 9.92 MB/sec execute 527 sec latency 58.590 ms 2024-04-12T21:56:08.028 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:07 smithi177 systemd[1]: Stopping Ceph osd.5 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:56:08.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:08 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[34005]: 2024-04-12T21:56:08.025+0000 7f3e6e9b5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:56:08.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:08 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[34005]: 2024-04-12T21:56:08.025+0000 7f3e6e9b5700 -1 osd.5 134 *** Got signal Terminated *** 2024-04-12T21:56:08.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:08 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[34005]: 2024-04-12T21:56:08.025+0000 7f3e6e9b5700 -1 osd.5 134 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T21:56:08.550 INFO:teuthology.orchestra.run.smithi123.stdout: 5 273077 9.92 MB/sec execute 528 sec latency 62.713 ms 2024-04-12T21:56:09.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:08 smithi123 ceph-mon[67226]: pgmap v243: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.2 MiB/s wr, 390 op/s; 1.6 MiB/s, 2 keys/s, 1 objects/s recovering 2024-04-12T21:56:09.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:08 smithi123 ceph-mon[67226]: osd.5 marked itself down and dead 2024-04-12T21:56:09.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:08 smithi177 ceph-mon[52869]: pgmap v243: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.2 MiB/s wr, 390 op/s; 1.6 MiB/s, 2 keys/s, 1 objects/s recovering 2024-04-12T21:56:09.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:08 smithi177 ceph-mon[52869]: osd.5 marked itself down and dead 2024-04-12T21:56:09.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 273199 9.91 MB/sec execute 529 sec latency 689.060 ms 2024-04-12T21:56:09.781 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:09 smithi177 podman[59627]: 2024-04-12 21:56:09.524521988 +0000 UTC m=+1.626665501 container died e55ce008c318469b348c752265e88a9d5b9e046bd0b8b6669706ae8ebc7094da (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, name=centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-12T21:56:10.140 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:09 smithi123 ceph-mon[67226]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:56:10.140 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:09 smithi123 ceph-mon[67226]: osdmap e135: 8 total, 7 up, 8 in 2024-04-12T21:56:10.140 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:09 smithi123 ceph-mon[67226]: osdmap e136: 8 total, 7 up, 8 in 2024-04-12T21:56:10.150 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:09 smithi177 ceph-mon[52869]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:56:10.150 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:09 smithi177 ceph-mon[52869]: osdmap e135: 8 total, 7 up, 8 in 2024-04-12T21:56:10.150 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:09 smithi177 ceph-mon[52869]: osdmap e136: 8 total, 7 up, 8 in 2024-04-12T21:56:10.151 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:10 smithi177 podman[59627]: 2024-04-12 21:56:10.134476158 +0000 UTC m=+2.236619671 container cleanup e55ce008c318469b348c752265e88a9d5b9e046bd0b8b6669706ae8ebc7094da (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream) 2024-04-12T21:56:10.151 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:10 smithi177 bash[59627]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5 2024-04-12T21:56:10.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 273522 9.91 MB/sec execute 530 sec latency 1041.980 ms 2024-04-12T21:56:10.777 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:10 smithi177 podman[59644]: 2024-04-12 21:56:10.394398134 +0000 UTC m=+0.864937594 container remove e55ce008c318469b348c752265e88a9d5b9e046bd0b8b6669706ae8ebc7094da (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.expose-services=, vcs-type=git, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870) 2024-04-12T21:56:10.904 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:10 smithi123 ceph-mon[67226]: pgmap v245: 97 pgs: 5 peering, 11 stale+active+clean, 81 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 466 op/s; 819 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-12T21:56:10.904 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:11.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:10 smithi177 ceph-mon[52869]: pgmap v245: 97 pgs: 5 peering, 11 stale+active+clean, 81 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 466 op/s; 819 KiB/s, 1 keys/s, 0 objects/s recovering 2024-04-12T21:56:11.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:11.031 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:10 smithi177 podman[59729]: 2024-04-12 21:56:10.681965184 +0000 UTC m=+0.022192335 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:56:11.031 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:10 smithi177 podman[59729]: 2024-04-12 21:56:10.803783665 +0000 UTC m=+0.144010823 container create d20485cbc9ccfbef25e70877a209ce928630b0d7073f16e1ab339d95ceead5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-12T21:56:11.448 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:11 smithi177 podman[59729]: 2024-04-12 21:56:11.081856244 +0000 UTC m=+0.422083409 container init d20485cbc9ccfbef25e70877a209ce928630b0d7073f16e1ab339d95ceead5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-b277feb) 2024-04-12T21:56:11.448 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:11 smithi177 podman[59729]: 2024-04-12 21:56:11.085216264 +0000 UTC m=+0.425443422 container start d20485cbc9ccfbef25e70877a209ce928630b0d7073f16e1ab339d95ceead5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T21:56:11.448 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:11 smithi177 podman[59729]: 2024-04-12 21:56:11.181893833 +0000 UTC m=+0.522120989 container attach d20485cbc9ccfbef25e70877a209ce928630b0d7073f16e1ab339d95ceead5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-12T21:56:11.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 273896 9.90 MB/sec execute 531 sec latency 72.985 ms 2024-04-12T21:56:11.781 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:11 smithi177 podman[59729]: 2024-04-12 21:56:11.447140056 +0000 UTC m=+0.787367207 container died d20485cbc9ccfbef25e70877a209ce928630b0d7073f16e1ab339d95ceead5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-12T21:56:12.177 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:11 smithi123 ceph-mon[67226]: Health check failed: Degraded data redundancy: 113/1650 objects degraded (6.848%), 12 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:12.262 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:11 smithi177 ceph-mon[52869]: Health check failed: Degraded data redundancy: 113/1650 objects degraded (6.848%), 12 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:12.517 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:12 smithi177 podman[59754]: 2024-04-12 21:56:12.26155047 +0000 UTC m=+0.807257942 container remove d20485cbc9ccfbef25e70877a209ce928630b0d7073f16e1ab339d95ceead5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-12T21:56:12.517 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:12 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-12T21:56:12.517 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:12 smithi177 systemd[1]: Stopped Ceph osd.5 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:56:12.518 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:12 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.5.service: Consumed 5min 53.157s CPU time. 2024-04-12T21:56:12.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 274304 9.90 MB/sec execute 532 sec latency 42.215 ms 2024-04-12T21:56:12.773 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:12 smithi177 systemd[1]: Starting Ceph osd.5 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:56:12.773 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:12 smithi177 podman[59850]: 2024-04-12 21:56:12.66051447 +0000 UTC m=+0.021904021 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:56:12.773 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:12 smithi177 podman[59850]: 2024-04-12 21:56:12.77119338 +0000 UTC m=+0.132582930 container create a9709e0f55ad31b2fd822348925976b43f4b2b5ff8aabfbb30c1fcf7533bd3a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240409) 2024-04-12T21:56:13.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:12 smithi177 ceph-mon[52869]: pgmap v247: 97 pgs: 5 active+undersized, 12 active+undersized+degraded, 5 peering, 4 stale+active+clean, 71 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 456 op/s; 113/1650 objects degraded (6.848%) 2024-04-12T21:56:13.198 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:12 smithi123 ceph-mon[67226]: pgmap v247: 97 pgs: 5 active+undersized, 12 active+undersized+degraded, 5 peering, 4 stale+active+clean, 71 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 456 op/s; 113/1650 objects degraded (6.848%) 2024-04-12T21:56:13.531 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 podman[59850]: 2024-04-12 21:56:13.040795987 +0000 UTC m=+0.402185540 container init a9709e0f55ad31b2fd822348925976b43f4b2b5ff8aabfbb30c1fcf7533bd3a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=) 2024-04-12T21:56:13.531 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 podman[59850]: 2024-04-12 21:56:13.045030726 +0000 UTC m=+0.406420279 container start a9709e0f55ad31b2fd822348925976b43f4b2b5ff8aabfbb30c1fcf7533bd3a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:56:13.531 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 podman[59850]: 2024-04-12 21:56:13.146236314 +0000 UTC m=+0.507625876 container attach a9709e0f55ad31b2fd822348925976b43f4b2b5ff8aabfbb30c1fcf7533bd3a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True) 2024-04-12T21:56:13.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 274605 9.90 MB/sec execute 533 sec latency 40.569 ms 2024-04-12T21:56:14.005 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate[59864]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T21:56:14.005 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate[59864]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T21:56:14.005 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 bash[59850]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T21:56:14.005 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 bash[59850]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 bash[59850]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 bash[59850]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 bash[59850]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 bash[59850]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 bash[59850]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate[59864]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate[59864]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate[59864]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate[59864]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate[59864]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-12T21:56:14.006 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:13 smithi177 podman[59850]: 2024-04-12 21:56:13.636678488 +0000 UTC m=+0.998068040 container died a9709e0f55ad31b2fd822348925976b43f4b2b5ff8aabfbb30c1fcf7533bd3a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240409) 2024-04-12T21:56:14.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 274929 9.91 MB/sec execute 534 sec latency 57.911 ms 2024-04-12T21:56:14.573 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:14 smithi177 podman[59997]: 2024-04-12 21:56:14.31239365 +0000 UTC m=+0.668648533 container remove a9709e0f55ad31b2fd822348925976b43f4b2b5ff8aabfbb30c1fcf7533bd3a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-activate, org.label-schema.build-date=20240409, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T21:56:14.573 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:14 smithi177 podman[60055]: 2024-04-12 21:56:14.446652669 +0000 UTC m=+0.021565115 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:56:14.573 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:14 smithi177 podman[60055]: 2024-04-12 21:56:14.571684284 +0000 UTC m=+0.146596710 container create 49cfbac67f9fb7d54c94b1595521838dc7e96c32acfb7f2a0724c51bb0de63ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=) 2024-04-12T21:56:15.010 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:14 smithi123 ceph-mon[67226]: pgmap v248: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 5 peering, 66 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.1 MiB/s wr, 462 op/s; 173/1662 objects degraded (10.409%) 2024-04-12T21:56:15.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:14 smithi177 ceph-mon[52869]: pgmap v248: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 5 peering, 66 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.1 MiB/s wr, 462 op/s; 173/1662 objects degraded (10.409%) 2024-04-12T21:56:15.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:14 smithi177 podman[60055]: 2024-04-12 21:56:14.826391669 +0000 UTC m=+0.401304098 container init 49cfbac67f9fb7d54c94b1595521838dc7e96c32acfb7f2a0724c51bb0de63ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T21:56:15.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:14 smithi177 podman[60055]: 2024-04-12 21:56:14.82997531 +0000 UTC m=+0.404887733 container start 49cfbac67f9fb7d54c94b1595521838dc7e96c32acfb7f2a0724c51bb0de63ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:56:15.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:14 smithi177 ceph-osd[60074]: -- 172.21.15.177:0/970900423 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fb1c087680 con 0x55fb1c051000 2024-04-12T21:56:15.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:14 smithi177 bash[60055]: 49cfbac67f9fb7d54c94b1595521838dc7e96c32acfb7f2a0724c51bb0de63ae 2024-04-12T21:56:15.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:14 smithi177 systemd[1]: Started Ceph osd.5 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:56:15.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 275247 9.91 MB/sec execute 535 sec latency 49.035 ms 2024-04-12T21:56:15.781 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:15 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T21:56:15.375+0000 7f19ddf567c0 -1 Falling back to public interface 2024-04-12T21:56:16.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:16.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:16.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:56:16.064 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:16.065 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:56:16.065 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:15 smithi123 ceph-mon[67226]: pgmap v249: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 504 op/s; 195/1659 objects degraded (11.754%) 2024-04-12T21:56:16.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:16.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:16.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:56:16.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:16.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:56:16.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:15 smithi177 ceph-mon[52869]: pgmap v249: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 504 op/s; 195/1659 objects degraded (11.754%) 2024-04-12T21:56:16.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 275572 9.90 MB/sec execute 536 sec latency 63.972 ms 2024-04-12T21:56:17.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 275897 9.89 MB/sec execute 537 sec latency 45.005 ms 2024-04-12T21:56:17.945 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:17 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 195/1668 objects degraded (11.691%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:18.099 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:17 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 195/1668 objects degraded (11.691%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:18.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 276254 9.89 MB/sec execute 538 sec latency 39.940 ms 2024-04-12T21:56:19.115 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:18 smithi123 ceph-mon[67226]: pgmap v250: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.0 MiB/s wr, 389 op/s; 195/1668 objects degraded (11.691%) 2024-04-12T21:56:19.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:18 smithi177 ceph-mon[52869]: pgmap v250: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 326 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.0 MiB/s wr, 389 op/s; 195/1668 objects degraded (11.691%) 2024-04-12T21:56:19.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 276557 9.89 MB/sec execute 539 sec latency 38.995 ms 2024-04-12T21:56:20.158 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:20 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:20.158 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:20 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:20.158 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:20 smithi123 ceph-mon[67226]: pgmap v251: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 524 op/s; 199/1662 objects degraded (11.974%) 2024-04-12T21:56:20.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:20 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:20.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:20 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:20.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:20 smithi177 ceph-mon[52869]: pgmap v251: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 524 op/s; 199/1662 objects degraded (11.974%) 2024-04-12T21:56:20.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 276910 9.90 MB/sec execute 540 sec latency 46.893 ms 2024-04-12T21:56:21.426 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:21.426 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:21.464 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:21.464 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:21.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 277251 9.89 MB/sec execute 541 sec latency 52.363 ms 2024-04-12T21:56:21.573 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:56:22.451 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:22 smithi123 ceph-mon[67226]: from='client.44289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:22.451 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:22 smithi123 ceph-mon[67226]: pgmap v252: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.1 MiB/s wr, 489 op/s; 184/1656 objects degraded (11.111%) 2024-04-12T21:56:22.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:22 smithi177 ceph-mon[52869]: from='client.44289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:22.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:22 smithi177 ceph-mon[52869]: pgmap v252: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.1 MiB/s wr, 489 op/s; 184/1656 objects degraded (11.111%) 2024-04-12T21:56:22.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 277510 9.88 MB/sec execute 542 sec latency 51.232 ms 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (8m) 82s ago 17m 23.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (4m) 82s ago 17m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (4m) 3s ago 16m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 82s ago 17m - - 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (11m) 82s ago 11m 38.2M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (11m) 3s ago 11m 15.5M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (7m) 82s ago 18m 510M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (6m) 3s ago 16m 452M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (6m) 82s ago 18m 49.7M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (5m) 3s ago 16m 45.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:56:22.592 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (11m) 82s ago 11m 283M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (11m) 3s ago 11m 87.4M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 82s ago 17m 19.3M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (7m) 3s ago 16m 17.3M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (3m) 82s ago 16m 332M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (3m) 82s ago 15m 283M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (2m) 82s ago 15m 220M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (92s) 82s ago 14m 13.5M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (47s) 3s ago 14m 334M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (8s) 3s ago 13m 12.2M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (13m) 3s ago 13m 600M 2704M 17.2.0 e1d6a67b021e 4c68828a2c8a 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (13m) 3s ago 13m 666M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:56:22.593 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (6m) 82s ago 16m 69.2M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:56:23.187 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 5 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:56:23.188 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:56:23.189 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-12T21:56:23.189 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 9 2024-04-12T21:56:23.189 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:56:23.189 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:56:23.468 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:23 smithi123 ceph-mon[67226]: from='client.44293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:23.468 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:23 smithi123 ceph-mon[67226]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:23.468 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:23 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3171259695' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:23.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 277749 9.87 MB/sec execute 543 sec latency 42.582 ms 2024-04-12T21:56:23.654 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "12/23 daemons upgraded", 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:56:23.655 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:56:23.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:23 smithi177 ceph-mon[52869]: from='client.44293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:23.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:23 smithi177 ceph-mon[52869]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:23.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:23 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3171259695' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 184/1656 objects degraded (11.111%), 23 pgs degraded 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout: osd.5 (root=default,host=smithi177) is down 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 184/1656 objects degraded (11.111%), 23 pgs degraded 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.b is active+undersized+degraded, acting [2,4] 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-04-12T21:56:24.171 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.12 is active+undersized+degraded, acting [3,2] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.15 is active+undersized+degraded, acting [1,4] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1e is active+undersized+degraded, acting [3,4] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-04-12T21:56:24.172 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-04-12T21:56:24.173 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-04-12T21:56:24.173 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-04-12T21:56:24.436 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T21:56:24.118+0000 7f19ddf567c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-12T21:56:24.520 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:24.520 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: pgmap v253: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 484 op/s; 178/1644 objects degraded (10.827%) 2024-04-12T21:56:24.520 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3267239195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:24.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T21:56:24.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 278027 9.87 MB/sec execute 544 sec latency 57.166 ms 2024-04-12T21:56:24.698 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:24.698 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: pgmap v253: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 484 op/s; 178/1644 objects degraded (10.827%) 2024-04-12T21:56:24.698 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3267239195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:56:24.698 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:24.698 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:24.698 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:56:24.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:56:24.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:24.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:24.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:56:24.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:24.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:24.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:24.699 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T21:56:25.551 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:25 smithi123 ceph-mon[67226]: pgmap v254: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 543 op/s; 177/1626 objects degraded (10.886%) 2024-04-12T21:56:25.551 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:25 smithi123 ceph-mon[67226]: pgmap v255: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.3 MiB/s wr, 553 op/s; 177/1626 objects degraded (10.886%) 2024-04-12T21:56:25.551 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:25 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T21:56:25.551 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:25 smithi123 ceph-mon[67226]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-12T21:56:25.551 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:25 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 177/1626 objects degraded (10.886%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:25.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 278307 9.87 MB/sec execute 545 sec latency 67.770 ms 2024-04-12T21:56:25.710 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:25 smithi177 ceph-mon[52869]: pgmap v254: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 543 op/s; 177/1626 objects degraded (10.886%) 2024-04-12T21:56:25.710 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:25 smithi177 ceph-mon[52869]: pgmap v255: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.3 MiB/s wr, 553 op/s; 177/1626 objects degraded (10.886%) 2024-04-12T21:56:25.710 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:25 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T21:56:25.710 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:25 smithi177 ceph-mon[52869]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-12T21:56:25.710 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:25 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 177/1626 objects degraded (10.886%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:26.031 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:25 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T21:56:25.707+0000 7f19ddf567c0 -1 osd.5 134 log_to_monitors true 2024-04-12T21:56:26.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 278678 9.88 MB/sec execute 546 sec latency 46.333 ms 2024-04-12T21:56:26.559 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:26 smithi123 ceph-mon[67226]: from='osd.5 [v2:172.21.15.177:6808/3545434944,v1:172.21.15.177:6809/3545434944]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T21:56:26.559 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:26 smithi123 ceph-mon[67226]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T21:56:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:26 smithi177 ceph-mon[52869]: from='osd.5 [v2:172.21.15.177:6808/3545434944,v1:172.21.15.177:6809/3545434944]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T21:56:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:26 smithi177 ceph-mon[52869]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T21:56:26.781 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 21:56:26 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T21:56:26.457+0000 7f19cf3ae700 -1 osd.5 134 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:56:27.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 279017 9.88 MB/sec execute 547 sec latency 100.213 ms 2024-04-12T21:56:27.568 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:27 smithi123 ceph-mon[67226]: pgmap v256: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 552 op/s; 172/1620 objects degraded (10.617%) 2024-04-12T21:56:27.568 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:27 smithi123 ceph-mon[67226]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T21:56:27.568 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:27 smithi123 ceph-mon[67226]: from='osd.5 [v2:172.21.15.177:6808/3545434944,v1:172.21.15.177:6809/3545434944]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:56:27.568 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:27 smithi123 ceph-mon[67226]: osdmap e137: 8 total, 7 up, 8 in 2024-04-12T21:56:27.568 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:27 smithi123 ceph-mon[67226]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:56:27.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:27 smithi177 ceph-mon[52869]: pgmap v256: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 552 op/s; 172/1620 objects degraded (10.617%) 2024-04-12T21:56:27.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:27 smithi177 ceph-mon[52869]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T21:56:27.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:27 smithi177 ceph-mon[52869]: from='osd.5 [v2:172.21.15.177:6808/3545434944,v1:172.21.15.177:6809/3545434944]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:56:27.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:27 smithi177 ceph-mon[52869]: osdmap e137: 8 total, 7 up, 8 in 2024-04-12T21:56:27.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:27 smithi177 ceph-mon[52869]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:56:28.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 279031 9.86 MB/sec execute 548 sec latency 1100.240 ms 2024-04-12T21:56:28.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:28 smithi123 ceph-mon[67226]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:56:28.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:28 smithi123 ceph-mon[67226]: osd.5 [v2:172.21.15.177:6808/3545434944,v1:172.21.15.177:6809/3545434944] boot 2024-04-12T21:56:28.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:28 smithi123 ceph-mon[67226]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T21:56:28.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:28 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:56:28.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:28 smithi123 ceph-mon[67226]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T21:56:28.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:28 smithi177 ceph-mon[52869]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:56:28.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:28 smithi177 ceph-mon[52869]: osd.5 [v2:172.21.15.177:6808/3545434944,v1:172.21.15.177:6809/3545434944] boot 2024-04-12T21:56:28.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:28 smithi177 ceph-mon[52869]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T21:56:28.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:28 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T21:56:28.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:28 smithi177 ceph-mon[52869]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T21:56:29.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 279135 9.84 MB/sec execute 549 sec latency 2100.283 ms 2024-04-12T21:56:29.708 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:29 smithi123 ceph-mon[67226]: pgmap v259: 97 pgs: 2 peering, 8 active+undersized, 21 active+undersized+degraded, 66 active+clean; 327 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 452 op/s; 156/1614 objects degraded (9.665%) 2024-04-12T21:56:29.708 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:29 smithi123 ceph-mon[67226]: osdmap e140: 8 total, 8 up, 8 in 2024-04-12T21:56:29.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:29 smithi177 ceph-mon[52869]: pgmap v259: 97 pgs: 2 peering, 8 active+undersized, 21 active+undersized+degraded, 66 active+clean; 327 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 452 op/s; 156/1614 objects degraded (9.665%) 2024-04-12T21:56:29.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:29 smithi177 ceph-mon[52869]: osdmap e140: 8 total, 8 up, 8 in 2024-04-12T21:56:30.466 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:30 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 156/1614 objects degraded (9.665%), 21 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:30.466 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:30.466 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:56:30.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 279426 9.83 MB/sec execute 550 sec latency 2241.429 ms 2024-04-12T21:56:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:30 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 156/1614 objects degraded (9.665%), 21 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:56:31.483 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:31 smithi123 ceph-mon[67226]: pgmap v262: 97 pgs: 1 remapped+peering, 2 peering, 6 active+undersized, 18 active+undersized+degraded, 70 active+clean; 327 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 389 op/s; 128/1629 objects degraded (7.858%); 38 KiB/s, 4 objects/s recovering 2024-04-12T21:56:31.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 279642 9.83 MB/sec execute 551 sec latency 72.220 ms 2024-04-12T21:56:31.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:31 smithi177 ceph-mon[52869]: pgmap v262: 97 pgs: 1 remapped+peering, 2 peering, 6 active+undersized, 18 active+undersized+degraded, 70 active+clean; 327 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 389 op/s; 128/1629 objects degraded (7.858%); 38 KiB/s, 4 objects/s recovering 2024-04-12T21:56:32.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 279797 9.82 MB/sec execute 552 sec latency 1029.985 ms 2024-04-12T21:56:32.725 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:32 smithi123 ceph-mon[67226]: osdmap e141: 8 total, 8 up, 8 in 2024-04-12T21:56:32.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:32 smithi177 ceph-mon[52869]: osdmap e141: 8 total, 8 up, 8 in 2024-04-12T21:56:33.508 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:33 smithi123 ceph-mon[67226]: pgmap v264: 97 pgs: 5 peering, 2 active+undersized+remapped, 90 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 525 op/s; 205 KiB/s, 40 objects/s recovering 2024-04-12T21:56:33.508 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:33 smithi123 ceph-mon[67226]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 128/1629 objects degraded (7.858%), 18 pgs degraded) 2024-04-12T21:56:33.508 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:33 smithi123 ceph-mon[67226]: osdmap e142: 8 total, 8 up, 8 in 2024-04-12T21:56:33.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 280170 9.82 MB/sec execute 553 sec latency 60.187 ms 2024-04-12T21:56:33.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:33 smithi177 ceph-mon[52869]: pgmap v264: 97 pgs: 5 peering, 2 active+undersized+remapped, 90 active+clean; 324 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 525 op/s; 205 KiB/s, 40 objects/s recovering 2024-04-12T21:56:33.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:33 smithi177 ceph-mon[52869]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 128/1629 objects degraded (7.858%), 18 pgs degraded) 2024-04-12T21:56:33.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:33 smithi177 ceph-mon[52869]: osdmap e142: 8 total, 8 up, 8 in 2024-04-12T21:56:34.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 280595 9.83 MB/sec execute 554 sec latency 42.047 ms 2024-04-12T21:56:35.508 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:35 smithi123 ceph-mon[67226]: pgmap v266: 97 pgs: 3 peering, 2 active+undersized+remapped, 92 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.4 MiB/s wr, 227 op/s; 300 KiB/s, 36 objects/s recovering 2024-04-12T21:56:35.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 280979 9.83 MB/sec execute 555 sec latency 39.228 ms 2024-04-12T21:56:35.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:35 smithi177 ceph-mon[52869]: pgmap v266: 97 pgs: 3 peering, 2 active+undersized+remapped, 92 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.4 MiB/s wr, 227 op/s; 300 KiB/s, 36 objects/s recovering 2024-04-12T21:56:36.551 INFO:teuthology.orchestra.run.smithi123.stdout: 5 281369 9.82 MB/sec execute 556 sec latency 55.405 ms 2024-04-12T21:56:37.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 281722 9.82 MB/sec execute 557 sec latency 45.988 ms 2024-04-12T21:56:37.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:37 smithi123 ceph-mon[67226]: pgmap v267: 97 pgs: 3 peering, 94 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.2 MiB/s wr, 291 op/s; 234 KiB/s, 28 objects/s recovering 2024-04-12T21:56:37.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:37 smithi177 ceph-mon[52869]: pgmap v267: 97 pgs: 3 peering, 94 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.2 MiB/s wr, 291 op/s; 234 KiB/s, 28 objects/s recovering 2024-04-12T21:56:38.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 282062 9.83 MB/sec execute 558 sec latency 42.818 ms 2024-04-12T21:56:39.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 282430 9.83 MB/sec execute 559 sec latency 90.769 ms 2024-04-12T21:56:39.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:39 smithi123 ceph-mon[67226]: pgmap v268: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 449 op/s; 218 KiB/s, 27 objects/s recovering 2024-04-12T21:56:39.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T21:56:39.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:39 smithi177 ceph-mon[52869]: pgmap v268: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 449 op/s; 218 KiB/s, 27 objects/s recovering 2024-04-12T21:56:39.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T21:56:40.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 282763 9.82 MB/sec execute 560 sec latency 69.420 ms 2024-04-12T21:56:40.656 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:40 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T21:56:40.656 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:40 smithi123 ceph-mon[67226]: Upgrade: osd.6 is safe to restart 2024-04-12T21:56:40.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:40 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T21:56:40.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:40 smithi177 ceph-mon[52869]: Upgrade: osd.6 is safe to restart 2024-04-12T21:56:41.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 283019 9.81 MB/sec execute 561 sec latency 61.058 ms 2024-04-12T21:56:41.630 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:41 smithi123 ceph-mon[67226]: pgmap v269: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.6 MiB/s wr, 316 op/s; 167 KiB/s, 19 objects/s recovering 2024-04-12T21:56:41.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:41.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T21:56:41.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:56:41.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:41 smithi177 ceph-mon[52869]: pgmap v269: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.6 MiB/s wr, 316 op/s; 167 KiB/s, 19 objects/s recovering 2024-04-12T21:56:41.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:41.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T21:56:41.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:56:42.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 283336 9.81 MB/sec execute 562 sec latency 59.798 ms 2024-04-12T21:56:42.684 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:42 smithi123 ceph-mon[67226]: Upgrade: Updating osd.6 2024-04-12T21:56:42.684 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:42 smithi123 ceph-mon[67226]: Deploying daemon osd.6 on smithi177 2024-04-12T21:56:42.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:42 smithi177 ceph-mon[52869]: Upgrade: Updating osd.6 2024-04-12T21:56:42.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:42 smithi177 ceph-mon[52869]: Deploying daemon osd.6 on smithi177 2024-04-12T21:56:43.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 283677 9.81 MB/sec execute 563 sec latency 49.525 ms 2024-04-12T21:56:43.556 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:43 smithi177 ceph-mon[52869]: pgmap v270: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 431 op/s; 91 KiB/s, 4 objects/s recovering 2024-04-12T21:56:43.715 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:43 smithi123 ceph-mon[67226]: pgmap v270: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 431 op/s; 91 KiB/s, 4 objects/s recovering 2024-04-12T21:56:44.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 283984 9.81 MB/sec execute 564 sec latency 45.043 ms 2024-04-12T21:56:45.162 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:44 smithi177 systemd[1]: Stopping Ceph osd.6 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:56:45.511 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:45 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[36996]: 2024-04-12T21:56:45.188+0000 7f03fe901700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:56:45.512 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:45 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[36996]: 2024-04-12T21:56:45.188+0000 7f03fe901700 -1 osd.6 142 *** Got signal Terminated *** 2024-04-12T21:56:45.512 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:45 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[36996]: 2024-04-12T21:56:45.188+0000 7f03fe901700 -1 osd.6 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T21:56:45.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 284261 9.81 MB/sec execute 565 sec latency 71.649 ms 2024-04-12T21:56:45.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:45 smithi123 ceph-mon[67226]: pgmap v271: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 474 op/s; 78 KiB/s, 3 objects/s recovering 2024-04-12T21:56:45.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:45.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:56:45.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:45 smithi123 ceph-mon[67226]: osd.6 marked itself down and dead 2024-04-12T21:56:45.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:45 smithi177 ceph-mon[52869]: pgmap v271: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 474 op/s; 78 KiB/s, 3 objects/s recovering 2024-04-12T21:56:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:56:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:45 smithi177 ceph-mon[52869]: osd.6 marked itself down and dead 2024-04-12T21:56:45.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:46.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 284476 9.80 MB/sec execute 566 sec latency 451.361 ms 2024-04-12T21:56:46.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:46 smithi177 ceph-mon[52869]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:56:46.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:46 smithi177 ceph-mon[52869]: osdmap e143: 8 total, 7 up, 8 in 2024-04-12T21:56:46.781 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:46 smithi177 podman[61833]: 2024-04-12 21:56:46.71434865 +0000 UTC m=+1.656841821 container died 4c68828a2c8a0b6c870bdc22368c5f85efa6c75df8a91b723c994c83808b674a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, io.openshift.expose-services=, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, RELEASE=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, maintainer=Guillaume Abrioux , vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container) 2024-04-12T21:56:46.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:46 smithi123 ceph-mon[67226]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:56:46.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:46 smithi123 ceph-mon[67226]: osdmap e143: 8 total, 7 up, 8 in 2024-04-12T21:56:47.374 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:47 smithi177 podman[61833]: 2024-04-12 21:56:47.115317571 +0000 UTC m=+2.057810735 container cleanup 4c68828a2c8a0b6c870bdc22368c5f85efa6c75df8a91b723c994c83808b674a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, maintainer=Guillaume Abrioux ) 2024-04-12T21:56:47.374 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:47 smithi177 bash[61833]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6 2024-04-12T21:56:47.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 284634 9.79 MB/sec execute 567 sec latency 1002.555 ms 2024-04-12T21:56:47.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:47 smithi123 ceph-mon[67226]: pgmap v273: 97 pgs: 10 stale+active+clean, 87 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 504 op/s; 16 KiB/s, 2 objects/s recovering 2024-04-12T21:56:47.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:47 smithi123 ceph-mon[67226]: osdmap e144: 8 total, 7 up, 8 in 2024-04-12T21:56:47.724 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:47 smithi177 ceph-mon[52869]: pgmap v273: 97 pgs: 10 stale+active+clean, 87 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 504 op/s; 16 KiB/s, 2 objects/s recovering 2024-04-12T21:56:47.724 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:47 smithi177 ceph-mon[52869]: osdmap e144: 8 total, 7 up, 8 in 2024-04-12T21:56:47.724 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:47 smithi177 podman[61893]: 2024-04-12 21:56:47.373103348 +0000 UTC m=+0.654317061 container remove 4c68828a2c8a0b6c870bdc22368c5f85efa6c75df8a91b723c994c83808b674a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, version=8, RELEASE=HEAD, GIT_CLEAN=True, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-12T21:56:48.003 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:47 smithi177 podman[61978]: 2024-04-12 21:56:47.627492848 +0000 UTC m=+0.019953102 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:56:48.003 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:47 smithi177 podman[61978]: 2024-04-12 21:56:47.749221118 +0000 UTC m=+0.141681350 container create f1c17d3434efe06f2cb97fe7bb30cd72d0253db544753e292dd8852e8cee4418 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-04-12T21:56:48.004 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:48 smithi177 podman[61978]: 2024-04-12 21:56:48.00230527 +0000 UTC m=+0.394765503 container init f1c17d3434efe06f2cb97fe7bb30cd72d0253db544753e292dd8852e8cee4418 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:56:48.281 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:48 smithi177 podman[61978]: 2024-04-12 21:56:48.006828725 +0000 UTC m=+0.399288962 container start f1c17d3434efe06f2cb97fe7bb30cd72d0253db544753e292dd8852e8cee4418 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-12T21:56:48.281 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:48 smithi177 podman[61978]: 2024-04-12 21:56:48.11064725 +0000 UTC m=+0.503107492 container attach f1c17d3434efe06f2cb97fe7bb30cd72d0253db544753e292dd8852e8cee4418 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, ceph=True, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T21:56:48.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 285039 9.79 MB/sec execute 568 sec latency 28.487 ms 2024-04-12T21:56:48.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:48 smithi123 ceph-mon[67226]: Health check failed: Degraded data redundancy: 42/1641 objects degraded (2.559%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:48.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:48 smithi177 ceph-mon[52869]: Health check failed: Degraded data redundancy: 42/1641 objects degraded (2.559%), 6 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:48.781 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:48 smithi177 podman[61978]: 2024-04-12 21:56:48.364717188 +0000 UTC m=+0.757177421 container died f1c17d3434efe06f2cb97fe7bb30cd72d0253db544753e292dd8852e8cee4418 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb) 2024-04-12T21:56:49.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 285416 9.79 MB/sec execute 569 sec latency 38.044 ms 2024-04-12T21:56:49.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:49 smithi123 ceph-mon[67226]: pgmap v275: 97 pgs: 2 active+undersized, 6 active+undersized+degraded, 8 peering, 5 stale+active+clean, 76 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 605 op/s; 42/1641 objects degraded (2.559%) 2024-04-12T21:56:49.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:49 smithi177 ceph-mon[52869]: pgmap v275: 97 pgs: 2 active+undersized, 6 active+undersized+degraded, 8 peering, 5 stale+active+clean, 76 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 605 op/s; 42/1641 objects degraded (2.559%) 2024-04-12T21:56:49.781 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:49 smithi177 podman[62004]: 2024-04-12 21:56:49.323601879 +0000 UTC m=+0.953084486 container remove f1c17d3434efe06f2cb97fe7bb30cd72d0253db544753e292dd8852e8cee4418 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T21:56:49.781 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:49 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-12T21:56:49.781 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:49 smithi177 systemd[1]: Stopped Ceph osd.6 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:56:49.781 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:49 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.6.service: Consumed 5min 23.719s CPU time. 2024-04-12T21:56:49.781 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:49 smithi177 systemd[1]: Starting Ceph osd.6 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:56:50.088 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:49 smithi177 podman[62108]: 2024-04-12 21:56:49.69804038 +0000 UTC m=+0.021664568 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:56:50.088 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:49 smithi177 podman[62108]: 2024-04-12 21:56:49.825419833 +0000 UTC m=+0.149044035 container create ddcb7b1e34d5a23891e4d5e67aadb17a8adb1e8158daba9ddc623f3cc58a6d6b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-12T21:56:50.088 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 podman[62108]: 2024-04-12 21:56:50.086584977 +0000 UTC m=+0.410209136 container init ddcb7b1e34d5a23891e4d5e67aadb17a8adb1e8158daba9ddc623f3cc58a6d6b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T21:56:50.531 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 podman[62108]: 2024-04-12 21:56:50.089955805 +0000 UTC m=+0.413579974 container start ddcb7b1e34d5a23891e4d5e67aadb17a8adb1e8158daba9ddc623f3cc58a6d6b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0) 2024-04-12T21:56:50.531 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 podman[62108]: 2024-04-12 21:56:50.186490989 +0000 UTC m=+0.510115166 container attach ddcb7b1e34d5a23891e4d5e67aadb17a8adb1e8158daba9ddc623f3cc58a6d6b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409) 2024-04-12T21:56:50.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 285808 9.80 MB/sec execute 570 sec latency 41.676 ms 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate[62122]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate[62122]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 bash[62108]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 bash[62108]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 bash[62108]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 bash[62108]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 bash[62108]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 bash[62108]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 bash[62108]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate[62122]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T21:56:50.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate[62122]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T21:56:50.968 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate[62122]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-12T21:56:50.968 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate[62122]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-12T21:56:50.968 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate[62122]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-12T21:56:50.968 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:50 smithi177 podman[62108]: 2024-04-12 21:56:50.665530011 +0000 UTC m=+0.989154180 container died ddcb7b1e34d5a23891e4d5e67aadb17a8adb1e8158daba9ddc623f3cc58a6d6b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T21:56:51.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 286227 9.80 MB/sec execute 571 sec latency 68.101 ms 2024-04-12T21:56:51.667 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:51 smithi123 ceph-mon[67226]: pgmap v276: 97 pgs: 3 active+undersized, 8 active+undersized+degraded, 8 peering, 5 stale+active+clean, 73 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.6 MiB/s wr, 397 op/s; 62/1635 objects degraded (3.792%) 2024-04-12T21:56:51.668 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:51 smithi177 ceph-mon[52869]: pgmap v276: 97 pgs: 3 active+undersized, 8 active+undersized+degraded, 8 peering, 5 stale+active+clean, 73 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.6 MiB/s wr, 397 op/s; 62/1635 objects degraded (3.792%) 2024-04-12T21:56:51.669 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:51 smithi177 podman[62249]: 2024-04-12 21:56:51.399546729 +0000 UTC m=+0.728692051 container remove ddcb7b1e34d5a23891e4d5e67aadb17a8adb1e8158daba9ddc623f3cc58a6d6b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-12T21:56:51.669 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:51 smithi177 podman[62309]: 2024-04-12 21:56:51.530913432 +0000 UTC m=+0.022722625 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:56:51.669 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:51 smithi177 podman[62309]: 2024-04-12 21:56:51.667575616 +0000 UTC m=+0.159384804 container create 7ce736a7262be52343df3a8619cefbe713abfdd026ec2e5766bdd4df6fceb8bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD) 2024-04-12T21:56:51.940 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:51 smithi177 podman[62309]: 2024-04-12 21:56:51.911853776 +0000 UTC m=+0.403662965 container init 7ce736a7262be52343df3a8619cefbe713abfdd026ec2e5766bdd4df6fceb8bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-12T21:56:51.940 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:51 smithi177 podman[62309]: 2024-04-12 21:56:51.916167131 +0000 UTC m=+0.407976338 container start 7ce736a7262be52343df3a8619cefbe713abfdd026ec2e5766bdd4df6fceb8bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-12T21:56:52.281 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:51 smithi177 ceph-osd[62328]: -- 172.21.15.177:0/1929669028 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5557b8595680 con 0x5557b855f000 2024-04-12T21:56:52.281 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:52 smithi177 bash[62309]: 7ce736a7262be52343df3a8619cefbe713abfdd026ec2e5766bdd4df6fceb8bd 2024-04-12T21:56:52.281 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:52 smithi177 systemd[1]: Started Ceph osd.6 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:56:52.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 286526 9.79 MB/sec execute 572 sec latency 37.017 ms 2024-04-12T21:56:52.781 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:52 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T21:56:52.458+0000 7faf698ac7c0 -1 Falling back to public interface 2024-04-12T21:56:53.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:53 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:53.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:53 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:53.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:53 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:56:53.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:53 smithi123 ceph-mon[67226]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 487 op/s; 201/1695 objects degraded (11.858%) 2024-04-12T21:56:53.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:53 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:53.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:53 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:53.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:53 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:56:53.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:53 smithi177 ceph-mon[52869]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 487 op/s; 201/1695 objects degraded (11.858%) 2024-04-12T21:56:53.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 286844 9.79 MB/sec execute 573 sec latency 44.562 ms 2024-04-12T21:56:54.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:54 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 201/1698 objects degraded (11.837%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:54.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 287233 9.79 MB/sec execute 574 sec latency 37.488 ms 2024-04-12T21:56:54.657 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:56:54.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:54 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 201/1698 objects degraded (11.837%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T21:56:55.500 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:55 smithi123 ceph-mon[67226]: pgmap v278: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.5 MiB/s wr, 457 op/s; 201/1698 objects degraded (11.837%) 2024-04-12T21:56:55.500 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:55 smithi123 ceph-mon[67226]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:55.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:55 smithi177 ceph-mon[52869]: pgmap v278: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.5 MiB/s wr, 457 op/s; 201/1698 objects degraded (11.837%) 2024-04-12T21:56:55.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:55 smithi177 ceph-mon[52869]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:55.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 287521 9.79 MB/sec execute 575 sec latency 48.674 ms 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (8m) 115s ago 17m 23.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (5m) 115s ago 17m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (5m) 36s ago 17m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 115s ago 17m - - 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (12m) 115s ago 12m 38.2M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (12m) 36s ago 12m 15.5M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (7m) 115s ago 19m 510M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:56:55.598 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (6m) 36s ago 17m 452M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (6m) 115s ago 19m 49.7M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (6m) 36s ago 17m 45.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (11m) 115s ago 11m 283M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (11m) 36s ago 11m 87.4M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (8m) 115s ago 17m 19.3M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (7m) 36s ago 17m 17.3M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (4m) 115s ago 16m 332M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (3m) 115s ago 16m 283M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (2m) 115s ago 15m 220M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (2m) 115s ago 15m 13.5M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (80s) 36s ago 14m 334M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (41s) 36s ago 14m 12.2M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 starting - - - 2704M 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (13m) 36s ago 13m 666M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:56:55.599 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (7m) 115s ago 16m 69.2M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 6 2024-04-12T21:56:56.159 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:56:56.160 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:56:56.160 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:56:56.160 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:56:56.160 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:56:56.160 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-12T21:56:56.160 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 10 2024-04-12T21:56:56.160 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:56:56.160 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:56:56.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:56 smithi123 ceph-mon[67226]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:56.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:56 smithi123 ceph-mon[67226]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:56.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:56 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1169975798' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:56.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:56 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:56.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:56 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:56.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 287829 9.79 MB/sec execute 576 sec latency 56.919 ms 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "13/23 daemons upgraded", 2024-04-12T21:56:56.622 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:56:56.623 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:56:56.623 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:56:56.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:56 smithi177 ceph-mon[52869]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:56.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:56 smithi177 ceph-mon[52869]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:56.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:56 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1169975798' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:56:56.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:56 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:56.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:56 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:57.223 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 196/1638 objects degraded (11.966%), 25 pgs degraded 2024-04-12T21:56:57.223 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:56:57.223 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:56:57.223 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T21:56:57.223 INFO:teuthology.orchestra.run.smithi123.stdout: osd.6 (root=default,host=smithi177) is down 2024-04-12T21:56:57.223 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 196/1638 objects degraded (11.966%), 25 pgs degraded 2024-04-12T21:56:57.223 INFO:teuthology.orchestra.run.smithi123.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.7 is active+undersized+degraded, acting [4,2] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1a is active+undersized+degraded, acting [4,1] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-04-12T21:56:57.224 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-04-12T21:56:57.225 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.a is active+undersized+degraded, acting [2,7] 2024-04-12T21:56:57.546 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:57 smithi123 ceph-mon[67226]: pgmap v279: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.7 MiB/s wr, 489 op/s; 196/1638 objects degraded (11.966%) 2024-04-12T21:56:57.546 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:57 smithi123 ceph-mon[67226]: from='client.44325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:57.547 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:57 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1640740051' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:56:57.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 288201 9.79 MB/sec execute 577 sec latency 60.463 ms 2024-04-12T21:56:57.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:57 smithi177 ceph-mon[52869]: pgmap v279: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.7 MiB/s wr, 489 op/s; 196/1638 objects degraded (11.966%) 2024-04-12T21:56:57.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:57 smithi177 ceph-mon[52869]: from='client.44325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:56:57.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:57 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1640740051' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:56:58.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 288579 9.78 MB/sec execute 578 sec latency 37.022 ms 2024-04-12T21:56:58.600 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:58.600 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:58.811 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:58.811 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:56:59.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 288952 9.79 MB/sec execute 579 sec latency 35.228 ms 2024-04-12T21:56:59.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:56:59 smithi177 ceph-mon[52869]: pgmap v280: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 600 op/s; 202/1656 objects degraded (12.198%) 2024-04-12T21:56:59.830 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:56:59 smithi123 ceph-mon[67226]: pgmap v280: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 600 op/s; 202/1656 objects degraded (12.198%) 2024-04-12T21:57:00.281 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:56:59 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T21:56:59.925+0000 7faf698ac7c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-12T21:57:00.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 289260 9.79 MB/sec execute 580 sec latency 103.464 ms 2024-04-12T21:57:00.594 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:00 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 202/1656 objects degraded (12.198%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:00.594 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:00.594 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:57:00.990 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:00 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 202/1656 objects degraded (12.198%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:00.990 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:00.990 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:57:01.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 289634 9.79 MB/sec execute 581 sec latency 63.570 ms 2024-04-12T21:57:01.600 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:01 smithi177 ceph-mon[52869]: pgmap v281: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 462 op/s; 200/1656 objects degraded (12.077%) 2024-04-12T21:57:01.600 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:01 smithi177 ceph-mon[52869]: from='osd.6 [v2:172.21.15.177:6816/3884470230,v1:172.21.15.177:6817/3884470230]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T21:57:01.600 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:01 smithi177 ceph-mon[52869]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T21:57:01.600 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:57:01 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T21:57:01.426+0000 7faf698ac7c0 -1 osd.6 142 log_to_monitors true 2024-04-12T21:57:01.636 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:01 smithi123 ceph-mon[67226]: pgmap v281: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 462 op/s; 200/1656 objects degraded (12.077%) 2024-04-12T21:57:01.637 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:01 smithi123 ceph-mon[67226]: from='osd.6 [v2:172.21.15.177:6816/3884470230,v1:172.21.15.177:6817/3884470230]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T21:57:01.637 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:01 smithi123 ceph-mon[67226]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T21:57:02.531 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 21:57:02 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T21:57:02.098+0000 7faf5ad04700 -1 osd.6 142 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:57:02.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 289928 9.78 MB/sec execute 582 sec latency 44.822 ms 2024-04-12T21:57:03.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:03 smithi123 ceph-mon[67226]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T21:57:03.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:03 smithi123 ceph-mon[67226]: osdmap e145: 8 total, 7 up, 8 in 2024-04-12T21:57:03.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:03 smithi123 ceph-mon[67226]: from='osd.6 [v2:172.21.15.177:6816/3884470230,v1:172.21.15.177:6817/3884470230]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:57:03.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:03 smithi123 ceph-mon[67226]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:57:03.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:03 smithi123 ceph-mon[67226]: pgmap v283: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 416 op/s; 201/1692 objects degraded (11.879%) 2024-04-12T21:57:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:03 smithi177 ceph-mon[52869]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T21:57:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:03 smithi177 ceph-mon[52869]: osdmap e145: 8 total, 7 up, 8 in 2024-04-12T21:57:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:03 smithi177 ceph-mon[52869]: from='osd.6 [v2:172.21.15.177:6816/3884470230,v1:172.21.15.177:6817/3884470230]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:57:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:03 smithi177 ceph-mon[52869]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:57:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:03 smithi177 ceph-mon[52869]: pgmap v283: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 416 op/s; 201/1692 objects degraded (11.879%) 2024-04-12T21:57:03.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 290240 9.78 MB/sec execute 583 sec latency 400.428 ms 2024-04-12T21:57:04.192 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: osd.6 [v2:172.21.15.177:6816/3884470230,v1:172.21.15.177:6817/3884470230] boot 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: osdmap e146: 8 total, 8 up, 8 in 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: pgmap v285: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 670 op/s; 199/1680 objects degraded (11.845%) 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: pgmap v286: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 525 op/s; 199/1680 objects degraded (11.845%) 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:04.193 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:57:04.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:04.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:04.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:04.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T21:57:04.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T21:57:04.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-12T21:57:04.194 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:04 smithi123 ceph-mon[67226]: osdmap e147: 8 total, 8 up, 8 in 2024-04-12T21:57:04.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:57:04.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: osd.6 [v2:172.21.15.177:6816/3884470230,v1:172.21.15.177:6817/3884470230] boot 2024-04-12T21:57:04.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: osdmap e146: 8 total, 8 up, 8 in 2024-04-12T21:57:04.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T21:57:04.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: pgmap v285: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 670 op/s; 199/1680 objects degraded (11.845%) 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: pgmap v286: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 525 op/s; 199/1680 objects degraded (11.845%) 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T21:57:04.532 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T21:57:04.533 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-12T21:57:04.533 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:04 smithi177 ceph-mon[52869]: osdmap e147: 8 total, 8 up, 8 in 2024-04-12T21:57:04.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 290438 9.77 MB/sec execute 584 sec latency 1031.660 ms 2024-04-12T21:57:05.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:05 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 199/1680 objects degraded (11.845%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:05.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:05 smithi123 ceph-mon[67226]: osdmap e148: 8 total, 8 up, 8 in 2024-04-12T21:57:05.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:05 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 199/1680 objects degraded (11.845%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:05.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:05 smithi177 ceph-mon[52869]: osdmap e148: 8 total, 8 up, 8 in 2024-04-12T21:57:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 290769 9.77 MB/sec execute 585 sec latency 773.235 ms 2024-04-12T21:57:06.257 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:06 smithi123 ceph-mon[67226]: pgmap v289: 97 pgs: 3 peering, 9 active+undersized, 23 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s wr, 649 op/s; 182/1695 objects degraded (10.737%) 2024-04-12T21:57:06.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:06 smithi177 ceph-mon[52869]: pgmap v289: 97 pgs: 3 peering, 9 active+undersized, 23 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s wr, 649 op/s; 182/1695 objects degraded (10.737%) 2024-04-12T21:57:06.554 INFO:teuthology.orchestra.run.smithi123.stdout: 5 291160 9.77 MB/sec execute 586 sec latency 104.758 ms 2024-04-12T21:57:07.460 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:07 smithi123 ceph-mon[67226]: osdmap e149: 8 total, 8 up, 8 in 2024-04-12T21:57:07.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:07 smithi177 ceph-mon[52869]: osdmap e149: 8 total, 8 up, 8 in 2024-04-12T21:57:07.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 291271 9.77 MB/sec execute 587 sec latency 791.115 ms 2024-04-12T21:57:08.218 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:08 smithi123 ceph-mon[67226]: pgmap v291: 97 pgs: 6 peering, 2 active+undersized, 2 active+undersized+remapped, 4 active+undersized+degraded, 83 active+clean; 334 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.2 MiB/s wr, 427 op/s; 38/1641 objects degraded (2.316%); 716 KiB/s, 44 objects/s recovering 2024-04-12T21:57:08.218 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:08 smithi123 ceph-mon[67226]: osdmap e150: 8 total, 8 up, 8 in 2024-04-12T21:57:08.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:08 smithi177 ceph-mon[52869]: pgmap v291: 97 pgs: 6 peering, 2 active+undersized, 2 active+undersized+remapped, 4 active+undersized+degraded, 83 active+clean; 334 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.2 MiB/s wr, 427 op/s; 38/1641 objects degraded (2.316%); 716 KiB/s, 44 objects/s recovering 2024-04-12T21:57:08.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:08 smithi177 ceph-mon[52869]: osdmap e150: 8 total, 8 up, 8 in 2024-04-12T21:57:08.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 291615 9.76 MB/sec execute 588 sec latency 56.417 ms 2024-04-12T21:57:09.499 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:09 smithi123 ceph-mon[67226]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/1641 objects degraded (2.316%), 4 pgs degraded) 2024-04-12T21:57:09.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:09 smithi177 ceph-mon[52869]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/1641 objects degraded (2.316%), 4 pgs degraded) 2024-04-12T21:57:09.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 291967 9.75 MB/sec execute 589 sec latency 49.885 ms 2024-04-12T21:57:10.373 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:10 smithi123 ceph-mon[67226]: pgmap v293: 97 pgs: 3 peering, 2 active+undersized+remapped, 92 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 362 op/s; 1.6 MiB/s, 53 keys/s, 44 objects/s recovering 2024-04-12T21:57:10.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:10 smithi177 ceph-mon[52869]: pgmap v293: 97 pgs: 3 peering, 2 active+undersized+remapped, 92 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 362 op/s; 1.6 MiB/s, 53 keys/s, 44 objects/s recovering 2024-04-12T21:57:10.552 INFO:teuthology.orchestra.run.smithi123.stdout: 5 292325 9.76 MB/sec execute 590 sec latency 41.521 ms 2024-04-12T21:57:11.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 292666 9.76 MB/sec execute 591 sec latency 112.292 ms 2024-04-12T21:57:12.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:12 smithi123 ceph-mon[67226]: pgmap v294: 97 pgs: 3 peering, 94 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.4 MiB/s wr, 356 op/s; 1.3 MiB/s, 44 keys/s, 37 objects/s recovering 2024-04-12T21:57:12.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:12 smithi177 ceph-mon[52869]: pgmap v294: 97 pgs: 3 peering, 94 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.4 MiB/s wr, 356 op/s; 1.3 MiB/s, 44 keys/s, 37 objects/s recovering 2024-04-12T21:57:12.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 293049 9.76 MB/sec execute 592 sec latency 36.987 ms 2024-04-12T21:57:13.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 293373 9.76 MB/sec execute 593 sec latency 49.319 ms 2024-04-12T21:57:14.454 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:14 smithi123 ceph-mon[67226]: pgmap v295: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 533 op/s; 2.5 MiB/s, 33 keys/s, 28 objects/s recovering 2024-04-12T21:57:14.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:14 smithi177 ceph-mon[52869]: pgmap v295: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 533 op/s; 2.5 MiB/s, 33 keys/s, 28 objects/s recovering 2024-04-12T21:57:14.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 293642 9.75 MB/sec execute 594 sec latency 38.350 ms 2024-04-12T21:57:15.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 293964 9.75 MB/sec execute 595 sec latency 42.502 ms 2024-04-12T21:57:16.242 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:16.242 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:57:16.242 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:16 smithi123 ceph-mon[67226]: pgmap v296: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 457 op/s; 2.1 MiB/s, 30 keys/s, 16 objects/s recovering 2024-04-12T21:57:16.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:16.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:57:16.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:16 smithi177 ceph-mon[52869]: pgmap v296: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 457 op/s; 2.1 MiB/s, 30 keys/s, 16 objects/s recovering 2024-04-12T21:57:16.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 294278 9.75 MB/sec execute 596 sec latency 49.046 ms 2024-04-12T21:57:17.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 294593 9.75 MB/sec execute 597 sec latency 54.227 ms 2024-04-12T21:57:18.284 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:18 smithi123 ceph-mon[67226]: pgmap v297: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 486 op/s; 1.7 MiB/s, 27 keys/s, 5 objects/s recovering 2024-04-12T21:57:18.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:18 smithi177 ceph-mon[52869]: pgmap v297: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 486 op/s; 1.7 MiB/s, 27 keys/s, 5 objects/s recovering 2024-04-12T21:57:18.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 294942 9.75 MB/sec execute 598 sec latency 49.620 ms 2024-04-12T21:57:19.327 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T21:57:19.327 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:19 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T21:57:19.327 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:19 smithi123 ceph-mon[67226]: Upgrade: osd.7 is safe to restart 2024-04-12T21:57:19.358 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T21:57:19.358 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:19 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T21:57:19.358 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:19 smithi177 ceph-mon[52869]: Upgrade: osd.7 is safe to restart 2024-04-12T21:57:19.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 295330 9.74 MB/sec execute 599 sec latency 50.158 ms 2024-04-12T21:57:20.341 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:20 smithi123 ceph-mon[67226]: pgmap v298: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 422 op/s; 1.4 MiB/s, 22 keys/s, 4 objects/s recovering 2024-04-12T21:57:20.341 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:20 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:20.341 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:20 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T21:57:20.341 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:20 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:57:20.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:20 smithi177 ceph-mon[52869]: pgmap v298: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 422 op/s; 1.4 MiB/s, 22 keys/s, 4 objects/s recovering 2024-04-12T21:57:20.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:20 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:20.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:20 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T21:57:20.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:20 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:57:20.553 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 600 sec 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: 0 cleanup 600 sec 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: Operation Count AvgLat MaxLat 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: ---------------------------------------- 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: NTCreateX 187874 4.538 1083.668 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: Close 137952 0.131 10.803 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: Rename 7955 9.733 112.283 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: Unlink 37955 7.135 1050.637 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: Qpathinfo 170410 1.130 1017.463 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: Qfileinfo 29699 0.008 1.328 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: Qfsinfo 31171 0.606 53.689 2024-04-12T21:57:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: Sfileinfo 15344 12.984 121.185 2024-04-12T21:57:20.821 INFO:teuthology.orchestra.run.smithi123.stdout: Find 65765 0.920 15.788 2024-04-12T21:57:20.821 INFO:teuthology.orchestra.run.smithi123.stdout: WriteX 92806 14.041 2336.813 2024-04-12T21:57:20.821 INFO:teuthology.orchestra.run.smithi123.stdout: ReadX 294144 0.010 5.359 2024-04-12T21:57:20.821 INFO:teuthology.orchestra.run.smithi123.stdout: LockX 610 0.368 4.654 2024-04-12T21:57:20.821 INFO:teuthology.orchestra.run.smithi123.stdout: UnlockX 610 0.318 3.107 2024-04-12T21:57:20.821 INFO:teuthology.orchestra.run.smithi123.stdout: Flush 13188 0.002 0.352 2024-04-12T21:57:20.821 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:57:20.821 INFO:teuthology.orchestra.run.smithi123.stdout:Throughput 9.74438 MB/sec 5 clients 5 procs max_latency=2336.821 ms 2024-04-12T21:57:20.822 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-04-12T21:57:20.857 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-04-12T21:57:20.991 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-04-12T21:57:21.018 INFO:teuthology.orchestra.run.smithi123.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-12T21:57:21.018 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:57:21.018 INFO:teuthology.orchestra.run.smithi123.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-04-12T21:57:21.019 INFO:teuthology.orchestra.run.smithi123.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-12T21:57:21.119 INFO:teuthology.orchestra.run.smithi123.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-12T21:57:21.119 INFO:teuthology.orchestra.run.smithi123.stdout:releasing clients 2024-04-12T21:57:21.637 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:21 smithi123 ceph-mon[67226]: Upgrade: Updating osd.7 2024-04-12T21:57:21.637 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:21 smithi123 ceph-mon[67226]: Deploying daemon osd.7 on smithi177 2024-04-12T21:57:21.637 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:21.769 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:21 smithi177 ceph-mon[52869]: Upgrade: Updating osd.7 2024-04-12T21:57:21.770 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:21 smithi177 ceph-mon[52869]: Deploying daemon osd.7 on smithi177 2024-04-12T21:57:21.770 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:22.633 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:22 smithi177 ceph-mon[52869]: pgmap v299: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.6 MiB/s wr, 429 op/s; 1024 KiB/s, 0 objects/s recovering 2024-04-12T21:57:22.659 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:22 smithi123 ceph-mon[67226]: pgmap v299: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.6 MiB/s wr, 429 op/s; 1024 KiB/s, 0 objects/s recovering 2024-04-12T21:57:23.119 INFO:teuthology.orchestra.run.smithi123.stdout: 5 299 18.99 MB/sec execute 1 sec latency 192.856 ms 2024-04-12T21:57:24.119 INFO:teuthology.orchestra.run.smithi123.stdout: 5 418 18.73 MB/sec execute 2 sec latency 67.869 ms 2024-04-12T21:57:24.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:24 smithi177 ceph-mon[52869]: pgmap v300: 97 pgs: 97 active+clean; 364 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.0 MiB/s wr, 573 op/s; 1024 KiB/s, 0 objects/s recovering 2024-04-12T21:57:24.531 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:24 smithi177 systemd[1]: Stopping Ceph osd.7 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:57:24.645 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:24 smithi123 ceph-mon[67226]: pgmap v300: 97 pgs: 97 active+clean; 364 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.0 MiB/s wr, 573 op/s; 1024 KiB/s, 0 objects/s recovering 2024-04-12T21:57:25.119 INFO:teuthology.orchestra.run.smithi123.stdout: 5 608 21.72 MB/sec execute 3 sec latency 71.510 ms 2024-04-12T21:57:25.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[40083]: 2024-04-12T21:57:24.790+0000 7f23fbce0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T21:57:25.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[40083]: 2024-04-12T21:57:24.790+0000 7f23fbce0700 -1 osd.7 150 *** Got signal Terminated *** 2024-04-12T21:57:25.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[40083]: 2024-04-12T21:57:24.790+0000 7f23fbce0700 -1 osd.7 150 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T21:57:25.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:25 smithi177 ceph-mon[52869]: osd.7 marked itself down and dead 2024-04-12T21:57:25.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:25 smithi123 ceph-mon[67226]: osd.7 marked itself down and dead 2024-04-12T21:57:26.119 INFO:teuthology.orchestra.run.smithi123.stdout: 5 721 17.48 MB/sec execute 4 sec latency 696.097 ms 2024-04-12T21:57:26.409 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:26 smithi177 podman[64107]: 2024-04-12 21:57:26.125832635 +0000 UTC m=+1.462397840 container died 2fe7fe8bf9c62e583290d4289c81d22f68a3a474106606fe31cb110a64720014 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-type=git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_CLEAN=True, version=8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-12T21:57:26.687 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:26 smithi123 ceph-mon[67226]: pgmap v301: 97 pgs: 97 active+clean; 357 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.0 MiB/s wr, 406 op/s 2024-04-12T21:57:26.687 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:26 smithi123 ceph-mon[67226]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:57:26.687 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:26 smithi123 ceph-mon[67226]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-12T21:57:26.687 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:26 smithi123 ceph-mon[67226]: osdmap e151: 8 total, 7 up, 8 in 2024-04-12T21:57:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:26 smithi177 ceph-mon[52869]: pgmap v301: 97 pgs: 97 active+clean; 357 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.0 MiB/s wr, 406 op/s 2024-04-12T21:57:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:26 smithi177 ceph-mon[52869]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T21:57:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:26 smithi177 ceph-mon[52869]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-12T21:57:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:26 smithi177 ceph-mon[52869]: osdmap e151: 8 total, 7 up, 8 in 2024-04-12T21:57:26.782 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:26 smithi177 podman[64107]: 2024-04-12 21:57:26.640776929 +0000 UTC m=+1.977342124 container cleanup 2fe7fe8bf9c62e583290d4289c81d22f68a3a474106606fe31cb110a64720014 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, name=centos-stream, distribution-scope=public, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vcs-type=git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64) 2024-04-12T21:57:26.782 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:26 smithi177 bash[64107]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7 2024-04-12T21:57:27.119 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 5 sec 2024-04-12T21:57:27.229 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:26 smithi177 podman[64156]: 2024-04-12 21:57:26.892214773 +0000 UTC m=+0.761991519 container remove 2fe7fe8bf9c62e583290d4289c81d22f68a3a474106606fe31cb110a64720014 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, version=8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, name=centos-stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870) 2024-04-12T21:57:27.328 INFO:teuthology.orchestra.run.smithi123.stdout: 0 cleanup 5 sec 2024-04-12T21:57:27.328 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:57:27.328 INFO:teuthology.orchestra.run.smithi123.stdout: Operation Count AvgLat MaxLat 2024-04-12T21:57:27.328 INFO:teuthology.orchestra.run.smithi123.stdout: ---------------------------------------- 2024-04-12T21:57:27.328 INFO:teuthology.orchestra.run.smithi123.stdout: NTCreateX 509 11.157 109.445 2024-04-12T21:57:27.328 INFO:teuthology.orchestra.run.smithi123.stdout: Close 478 11.827 86.842 2024-04-12T21:57:27.328 INFO:teuthology.orchestra.run.smithi123.stdout: Rename 15 229.440 1055.493 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: Unlink 37 3.055 16.427 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: Qpathinfo 110 1.345 24.937 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: Qfileinfo 320 18.426 94.073 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: Qfsinfo 79 0.437 1.995 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: Sfileinfo 68 27.640 1041.651 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: Find 99 0.721 1.221 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: WriteX 1530 0.026 0.069 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: ReadX 359 0.018 0.379 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: Flush 102 21.612 1040.485 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T21:57:27.329 INFO:teuthology.orchestra.run.smithi123.stdout:Throughput 17.4786 MB/sec 5 clients 5 procs max_latency=696.097 ms 2024-04-12T21:57:27.330 DEBUG:teuthology.parallel:result is None 2024-04-12T21:57:27.522 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:27 smithi177 ceph-mon[52869]: osdmap e152: 8 total, 7 up, 8 in 2024-04-12T21:57:27.522 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:27 smithi177 podman[64247]: 2024-04-12 21:57:27.131059002 +0000 UTC m=+0.019513242 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:57:27.522 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:27 smithi177 podman[64247]: 2024-04-12 21:57:27.259888588 +0000 UTC m=+0.148342855 container create db6acb6ddd3ef4f02e419038a94ab96e16ab47843572da651d7439f6cce55e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-12T21:57:27.605 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:57:27.781 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:27 smithi177 podman[64247]: 2024-04-12 21:57:27.521022994 +0000 UTC m=+0.409477222 container init db6acb6ddd3ef4f02e419038a94ab96e16ab47843572da651d7439f6cce55e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, ceph=True) 2024-04-12T21:57:27.781 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:27 smithi177 podman[64247]: 2024-04-12 21:57:27.524539328 +0000 UTC m=+0.412993550 container start db6acb6ddd3ef4f02e419038a94ab96e16ab47843572da651d7439f6cce55e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T21:57:27.781 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:27 smithi177 podman[64247]: 2024-04-12 21:57:27.629410861 +0000 UTC m=+0.517865095 container attach db6acb6ddd3ef4f02e419038a94ab96e16ab47843572da651d7439f6cce55e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-b277feb) 2024-04-12T21:57:27.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:27 smithi123 ceph-mon[67226]: osdmap e152: 8 total, 7 up, 8 in 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (9m) 2m ago 18m 23.9M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (5m) 2m ago 18m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (5m) 32s ago 17m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 2m ago 18m - - 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (12m) 2m ago 12m 38.2M - 17.2.0 e1d6a67b021e e047db68b186 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (12m) 32s ago 12m 15.7M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (8m) 2m ago 19m 510M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (7m) 32s ago 17m 453M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:57:28.260 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (7m) 2m ago 19m 49.7M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (6m) 32s ago 17m 44.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (12m) 2m ago 12m 283M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (12m) 32s ago 12m 87.4M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (9m) 2m ago 18m 19.3M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (8m) 32s ago 17m 17.4M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (5m) 2m ago 17m 332M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (4m) 2m ago 16m 283M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (3m) 2m ago 16m 220M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (2m) 2m ago 15m 13.5M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (113s) 32s ago 15m 379M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (73s) 32s ago 14m 257M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (36s) 32s ago 14m 12.6M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7ce736a7262b 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (14m) 32s ago 14m 696M 2704M 17.2.0 e1d6a67b021e 2fe7fe8bf9c6 2024-04-12T21:57:28.261 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (7m) 2m ago 17m 69.2M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:57:28.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:27 smithi177 podman[64247]: 2024-04-12 21:57:27.816631401 +0000 UTC m=+0.705085628 container died db6acb6ddd3ef4f02e419038a94ab96e16ab47843572da651d7439f6cce55e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-12T21:57:28.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:28 smithi177 ceph-mon[52869]: pgmap v304: 97 pgs: 6 active+undersized, 13 active+undersized+degraded, 3 peering, 75 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s wr, 445 op/s; 61/1341 objects degraded (4.549%) 2024-04-12T21:57:28.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:28 smithi177 ceph-mon[52869]: Health check failed: Degraded data redundancy: 61/1341 objects degraded (4.549%), 13 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:28.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:28 smithi177 ceph-mon[52869]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:57:28.779 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:28 smithi177 ceph-mon[52869]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:57:28.779 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:28 smithi177 podman[64273]: 2024-04-12 21:57:28.466586337 +0000 UTC m=+0.644662926 container remove db6acb6ddd3ef4f02e419038a94ab96e16ab47843572da651d7439f6cce55e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240409) 2024-04-12T21:57:28.779 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:28 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-12T21:57:28.779 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:28 smithi177 systemd[1]: Stopped Ceph osd.7 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:57:28.780 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:28 smithi177 systemd[1]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.7.service: Consumed 8min 20.218s CPU time. 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 7 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:57:28.783 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-12T21:57:28.784 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 11 2024-04-12T21:57:28.784 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:57:28.784 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:57:28.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:28 smithi123 ceph-mon[67226]: pgmap v304: 97 pgs: 6 active+undersized, 13 active+undersized+degraded, 3 peering, 75 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s wr, 445 op/s; 61/1341 objects degraded (4.549%) 2024-04-12T21:57:28.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:28 smithi123 ceph-mon[67226]: Health check failed: Degraded data redundancy: 61/1341 objects degraded (4.549%), 13 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:28.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:28 smithi123 ceph-mon[67226]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:57:28.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:28 smithi123 ceph-mon[67226]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:57:29.031 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:28 smithi177 systemd[1]: Starting Ceph osd.7 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T21:57:29.031 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:28 smithi177 podman[64381]: 2024-04-12 21:57:28.896980339 +0000 UTC m=+0.018194911 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "crash" 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "13/23 daemons upgraded", 2024-04-12T21:57:29.119 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T21:57:29.120 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:57:29.120 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:57:29.497 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-mon[52869]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:57:29.497 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/616594200' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:29.497 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 podman[64381]: 2024-04-12 21:57:29.243100971 +0000 UTC m=+0.364315568 container create d1b9a3f432308665c2b523636b23ac970f5940c3d34a5561d2ba6cb2e7f383d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T21:57:29.518 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 92/1233 objects degraded (7.461%), 21 pgs degraded 2024-04-12T21:57:29.518 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:57:29.518 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:57:29.518 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T21:57:29.518 INFO:teuthology.orchestra.run.smithi123.stdout: osd.7 (root=default,host=smithi177) is down 2024-04-12T21:57:29.518 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-04-12T21:57:29.518 INFO:teuthology.orchestra.run.smithi123.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-04-12T21:57:29.518 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 92/1233 objects degraded (7.461%), 21 pgs degraded 2024-04-12T21:57:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-04-12T21:57:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-04-12T21:57:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-04-12T21:57:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-04-12T21:57:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-04-12T21:57:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-04-12T21:57:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-04-12T21:57:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-04-12T21:57:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-04-12T21:57:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-04-12T21:57:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-04-12T21:57:29.521 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-04-12T21:57:29.521 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-04-12T21:57:29.521 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-04-12T21:57:29.521 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-04-12T21:57:29.521 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-04-12T21:57:29.521 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-04-12T21:57:29.521 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-04-12T21:57:29.522 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-04-12T21:57:29.522 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.a is active+undersized+degraded, acting [6,2] 2024-04-12T21:57:29.522 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-04-12T21:57:29.781 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 podman[64381]: 2024-04-12 21:57:29.495446354 +0000 UTC m=+0.616660907 container init d1b9a3f432308665c2b523636b23ac970f5940c3d34a5561d2ba6cb2e7f383d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD) 2024-04-12T21:57:29.781 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 podman[64381]: 2024-04-12 21:57:29.498562909 +0000 UTC m=+0.619777471 container start d1b9a3f432308665c2b523636b23ac970f5940c3d34a5561d2ba6cb2e7f383d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240409, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T21:57:29.781 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 podman[64381]: 2024-04-12 21:57:29.587061412 +0000 UTC m=+0.708275966 container attach d1b9a3f432308665c2b523636b23ac970f5940c3d34a5561d2ba6cb2e7f383d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T21:57:29.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:29 smithi123 ceph-mon[67226]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:57:29.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:29 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/616594200' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:30.275 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate[64396]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-12T21:57:30.275 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate[64396]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T21:57:30.275 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 bash[64381]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-12T21:57:30.275 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 bash[64381]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T21:57:30.275 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 bash[64381]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 bash[64381]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 bash[64381]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 bash[64381]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 bash[64381]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate[64396]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate[64396]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate[64396]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate[64396]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate[64396]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-12T21:57:30.276 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:29 smithi177 podman[64381]: 2024-04-12 21:57:29.967329693 +0000 UTC m=+1.088544255 container died d1b9a3f432308665c2b523636b23ac970f5940c3d34a5561d2ba6cb2e7f383d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240409) 2024-04-12T21:57:30.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:30 smithi177 ceph-mon[52869]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:57:30.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:30 smithi177 ceph-mon[52869]: pgmap v305: 97 pgs: 8 active+undersized, 21 active+undersized+degraded, 3 peering, 65 active+clean; 315 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 485 op/s; 92/1233 objects degraded (7.461%) 2024-04-12T21:57:30.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:30 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1719174891' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:57:30.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:30.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:57:30.873 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:30 smithi177 podman[64512]: 2024-04-12 21:57:30.532450917 +0000 UTC m=+0.557802876 container remove d1b9a3f432308665c2b523636b23ac970f5940c3d34a5561d2ba6cb2e7f383d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T21:57:30.873 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:30 smithi177 podman[64579]: 2024-04-12 21:57:30.645777626 +0000 UTC m=+0.017865093 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T21:57:30.873 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:30 smithi177 podman[64579]: 2024-04-12 21:57:30.758551925 +0000 UTC m=+0.130639382 container create 5bea7c19fe20273be90100e462bdf06c404320e94ec690172890f47ee83e701c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-12T21:57:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:30 smithi123 ceph-mon[67226]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:57:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:30 smithi123 ceph-mon[67226]: pgmap v305: 97 pgs: 8 active+undersized, 21 active+undersized+degraded, 3 peering, 65 active+clean; 315 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s wr, 485 op/s; 92/1233 objects degraded (7.461%) 2024-04-12T21:57:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:30 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1719174891' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:57:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:57:31.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:30 smithi177 podman[64579]: 2024-04-12 21:57:30.978269846 +0000 UTC m=+0.350357306 container init 5bea7c19fe20273be90100e462bdf06c404320e94ec690172890f47ee83e701c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.license=GPLv2) 2024-04-12T21:57:31.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:30 smithi177 podman[64579]: 2024-04-12 21:57:30.981609046 +0000 UTC m=+0.353696504 container start 5bea7c19fe20273be90100e462bdf06c404320e94ec690172890f47ee83e701c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, GIT_CLEAN=True) 2024-04-12T21:57:31.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:31 smithi177 ceph-osd[64598]: -- 172.21.15.177:0/2670581382 <== mon.1 v2:172.21.15.177:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55df0be93680 con 0x55df0be5d000 2024-04-12T21:57:31.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:31 smithi177 bash[64579]: 5bea7c19fe20273be90100e462bdf06c404320e94ec690172890f47ee83e701c 2024-04-12T21:57:31.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:31 smithi177 systemd[1]: Started Ceph osd.7 for c8b0510a-f914-11ee-bc8d-c7b262605968. 2024-04-12T21:57:31.781 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:31 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T21:57:31.528+0000 7f898a26e7c0 -1 Falling back to public interface 2024-04-12T21:57:32.409 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:32.409 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:32.409 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:57:32.410 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:32 smithi177 ceph-mon[52869]: pgmap v306: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 61 active+clean; 295 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s wr, 279 op/s; 99/996 objects degraded (9.940%); 925 B/s, 0 objects/s recovering 2024-04-12T21:57:32.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:32.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:32.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:57:32.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:32 smithi123 ceph-mon[67226]: pgmap v306: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 61 active+clean; 295 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s wr, 279 op/s; 99/996 objects degraded (9.940%); 925 B/s, 0 objects/s recovering 2024-04-12T21:57:33.302 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:33 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 54/480 objects degraded (11.250%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:33.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:33 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 54/480 objects degraded (11.250%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:34.573 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:34 smithi177 ceph-mon[52869]: pgmap v307: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 61 active+clean; 237 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s wr, 294 op/s; 54/480 objects degraded (11.250%); 925 B/s, 0 objects/s recovering 2024-04-12T21:57:34.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:34 smithi123 ceph-mon[67226]: pgmap v307: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 61 active+clean; 237 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s wr, 294 op/s; 54/480 objects degraded (11.250%); 925 B/s, 0 objects/s recovering 2024-04-12T21:57:36.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:36.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:36 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:36.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:36 smithi123 ceph-mon[67226]: pgmap v308: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 61 active+clean; 235 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 MiB/s wr, 242 op/s; 54/465 objects degraded (11.613%); 760 B/s, 0 objects/s recovering 2024-04-12T21:57:36.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:36 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:36.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:36 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:36.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:36 smithi177 ceph-mon[52869]: pgmap v308: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 61 active+clean; 235 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 MiB/s wr, 242 op/s; 54/465 objects degraded (11.613%); 760 B/s, 0 objects/s recovering 2024-04-12T21:57:37.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:37.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:37.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:37.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:38.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:38 smithi177 ceph-mon[52869]: pgmap v309: 97 pgs: 10 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s wr, 156 op/s; 51/426 objects degraded (11.972%); 689 B/s, 0 objects/s recovering 2024-04-12T21:57:38.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:38 smithi123 ceph-mon[67226]: pgmap v309: 97 pgs: 10 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s wr, 156 op/s; 51/426 objects degraded (11.972%); 689 B/s, 0 objects/s recovering 2024-04-12T21:57:40.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:39 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T21:57:39.785+0000 7f898a26e7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: pgmap v310: 97 pgs: 10 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.7 MiB/s wr, 106 op/s; 51/426 objects degraded (11.972%); 617 B/s, 0 objects/s recovering 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 51/426 objects degraded (11.972%), 26 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: pgmap v311: 97 pgs: 10 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 342 KiB/s wr, 61 op/s; 51/426 objects degraded (11.972%); 690 B/s, 0 objects/s recovering 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: pgmap v312: 97 pgs: 10 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 87 KiB/s wr, 29 op/s; 51/426 objects degraded (11.972%) 2024-04-12T21:57:40.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-12T21:57:40.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-12T21:57:40.672 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-12T21:57:40.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: pgmap v310: 97 pgs: 10 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.7 MiB/s wr, 106 op/s; 51/426 objects degraded (11.972%); 617 B/s, 0 objects/s recovering 2024-04-12T21:57:40.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 51/426 objects degraded (11.972%), 26 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: pgmap v311: 97 pgs: 10 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 342 KiB/s wr, 61 op/s; 51/426 objects degraded (11.972%); 690 B/s, 0 objects/s recovering 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: pgmap v312: 97 pgs: 10 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 87 KiB/s wr, 29 op/s; 51/426 objects degraded (11.972%) 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:40.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-12T21:57:40.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-12T21:57:40.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-12T21:57:40.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-12T21:57:40.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-12T21:57:40.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-12T21:57:40.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-12T21:57:40.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-12T21:57:41.531 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T21:57:41.043+0000 7f898a26e7c0 -1 osd.7 150 log_to_monitors true 2024-04-12T21:57:41.531 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T21:57:41.089+0000 7f897b6c6700 -1 osd.7 150 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all osd 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: Upgrade: Setting require_osd_release to 19 squid 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: from='osd.7 [v2:172.21.15.177:6824/3375298453,v1:172.21.15.177:6825/3375298453]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: from='osd.7 [v2:172.21.15.177:6824/3375298453,v1:172.21.15.177:6825/3375298453]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:57:41.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: osdmap e153: 8 total, 7 up, 8 in 2024-04-12T21:57:41.921 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:57:41.921 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi123.poxexz"]}]: dispatch 2024-04-12T21:57:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all osd 2024-04-12T21:57:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: Upgrade: Setting require_osd_release to 19 squid 2024-04-12T21:57:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: from='osd.7 [v2:172.21.15.177:6824/3375298453,v1:172.21.15.177:6825/3375298453]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T21:57:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T21:57:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-12T21:57:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-12T21:57:42.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T21:57:42.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: from='osd.7 [v2:172.21.15.177:6824/3375298453,v1:172.21.15.177:6825/3375298453]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:57:42.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: osdmap e153: 8 total, 7 up, 8 in 2024-04-12T21:57:42.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi177", "root=default"]}]: dispatch 2024-04-12T21:57:42.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi123.poxexz"]}]: dispatch 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: Upgrade: It appears safe to stop mds.foofs.smithi123.poxexz 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: pgmap v314: 97 pgs: 11 active+undersized, 25 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 760 B/s rd, 44 KiB/s wr, 5 op/s; 51/426 objects degraded (11.972%) 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: osd.7 [v2:172.21.15.177:6824/3375298453,v1:172.21.15.177:6825/3375298453] boot 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: osdmap e154: 8 total, 8 up, 8 in 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.poxexz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:57:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:57:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: Upgrade: It appears safe to stop mds.foofs.smithi123.poxexz 2024-04-12T21:57:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: pgmap v314: 97 pgs: 11 active+undersized, 25 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 760 B/s rd, 44 KiB/s wr, 5 op/s; 51/426 objects degraded (11.972%) 2024-04-12T21:57:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T21:57:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: osd.7 [v2:172.21.15.177:6824/3375298453,v1:172.21.15.177:6825/3375298453] boot 2024-04-12T21:57:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: osdmap e154: 8 total, 8 up, 8 in 2024-04-12T21:57:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T21:57:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.poxexz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:57:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:57:44.182 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:43 smithi177 ceph-mon[52869]: Upgrade: Updating mds.foofs.smithi123.poxexz 2024-04-12T21:57:44.182 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:43 smithi177 ceph-mon[52869]: Deploying daemon mds.foofs.smithi123.poxexz on smithi123 2024-04-12T21:57:44.183 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:43 smithi177 ceph-mon[52869]: osdmap e155: 8 total, 8 up, 8 in 2024-04-12T21:57:44.408 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:43 smithi123 ceph-mon[67226]: Upgrade: Updating mds.foofs.smithi123.poxexz 2024-04-12T21:57:44.408 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:43 smithi123 ceph-mon[67226]: Deploying daemon mds.foofs.smithi123.poxexz on smithi123 2024-04-12T21:57:44.408 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:43 smithi123 ceph-mon[67226]: osdmap e155: 8 total, 8 up, 8 in 2024-04-12T21:57:45.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:44 smithi123 ceph-mon[67226]: pgmap v317: 97 pgs: 9 peering, 9 active+undersized, 18 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s wr, 0 op/s; 31/426 objects degraded (7.277%) 2024-04-12T21:57:45.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:44 smithi123 ceph-mon[67226]: osdmap e156: 8 total, 8 up, 8 in 2024-04-12T21:57:45.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:44 smithi123 ceph-mon[67226]: Health check update: Degraded data redundancy: 31/426 objects degraded (7.277%), 18 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:45.021 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:44 smithi123 ceph-mon[67226]: osdmap e157: 8 total, 8 up, 8 in 2024-04-12T21:57:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:44 smithi177 ceph-mon[52869]: pgmap v317: 97 pgs: 9 peering, 9 active+undersized, 18 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s wr, 0 op/s; 31/426 objects degraded (7.277%) 2024-04-12T21:57:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:44 smithi177 ceph-mon[52869]: osdmap e156: 8 total, 8 up, 8 in 2024-04-12T21:57:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:44 smithi177 ceph-mon[52869]: Health check update: Degraded data redundancy: 31/426 objects degraded (7.277%), 18 pgs degraded (PG_DEGRADED) 2024-04-12T21:57:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:44 smithi177 ceph-mon[52869]: osdmap e157: 8 total, 8 up, 8 in 2024-04-12T21:57:45.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[67221]: 2024-04-12T21:57:45.017+0000 7f13a5559700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:57:45.420 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T21:57:45.053+0000 7f0fd4bdf700 -1 bdev(0x555b2e6e2000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-04-12T21:57:45.420 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T21:57:45.055+0000 7f0fd33dc700 -1 bdev(0x555b2e6e2000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: osdmap e158: 8 total, 8 up, 8 in 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: Standby daemon mds.foofs.smithi177.hqctzz assigned to filesystem foofs as rank 0 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: fsmap foofs:1/1 {0=foofs.smithi177.hqctzz=up:replay} 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:46.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:57:46.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-12T21:57:46.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:57:46.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: osdmap e158: 8 total, 8 up, 8 in 2024-04-12T21:57:46.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: Standby daemon mds.foofs.smithi177.hqctzz assigned to filesystem foofs as rank 0 2024-04-12T21:57:46.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-12T21:57:46.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T21:57:46.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-12T21:57:46.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: fsmap foofs:1/1 {0=foofs.smithi177.hqctzz=up:replay} 2024-04-12T21:57:46.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:46.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:57:47.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:46 smithi177 ceph-mon[52869]: pgmap v321: 97 pgs: 9 peering, 9 active+undersized, 12 active+undersized+degraded, 67 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 532 B/s wr, 1 op/s; 16/426 objects degraded (3.756%); 643 KiB/s, 4 objects/s recovering 2024-04-12T21:57:47.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:46 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:57:47.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:46 smithi123 ceph-mon[67226]: pgmap v321: 97 pgs: 9 peering, 9 active+undersized, 12 active+undersized+degraded, 67 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 532 B/s wr, 1 op/s; 16/426 objects degraded (3.756%); 643 KiB/s, 4 objects/s recovering 2024-04-12T21:57:47.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:46 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:57:48.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:48 smithi123 ceph-mon[67226]: reconnect by client.24323 172.21.15.177:0/3783509255 after 0 2024-04-12T21:57:48.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:48 smithi123 ceph-mon[67226]: reconnect by client.14548 172.21.15.123:0/3378876472 after 0 2024-04-12T21:57:48.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:48 smithi123 ceph-mon[67226]: mds.? [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] up:reconnect 2024-04-12T21:57:48.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:48 smithi123 ceph-mon[67226]: fsmap foofs:1/1 {0=foofs.smithi177.hqctzz=up:reconnect} 2024-04-12T21:57:48.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:48 smithi123 ceph-mon[67226]: pgmap v322: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 422 B/s wr, 11 op/s; 511 KiB/s, 21 objects/s recovering 2024-04-12T21:57:48.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:48 smithi177 ceph-mon[52869]: reconnect by client.24323 172.21.15.177:0/3783509255 after 0 2024-04-12T21:57:48.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:48 smithi177 ceph-mon[52869]: reconnect by client.14548 172.21.15.123:0/3378876472 after 0 2024-04-12T21:57:48.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:48 smithi177 ceph-mon[52869]: mds.? [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] up:reconnect 2024-04-12T21:57:48.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:48 smithi177 ceph-mon[52869]: fsmap foofs:1/1 {0=foofs.smithi177.hqctzz=up:reconnect} 2024-04-12T21:57:48.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:48 smithi177 ceph-mon[52869]: pgmap v322: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 422 B/s wr, 11 op/s; 511 KiB/s, 21 objects/s recovering 2024-04-12T21:57:49.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:49 smithi123 ceph-mon[67226]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/426 objects degraded (3.756%), 12 pgs degraded) 2024-04-12T21:57:49.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:49 smithi123 ceph-mon[67226]: mds.? [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] up:rejoin 2024-04-12T21:57:49.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:49 smithi123 ceph-mon[67226]: fsmap foofs:1/1 {0=foofs.smithi177.hqctzz=up:rejoin} 2024-04-12T21:57:49.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:49 smithi123 ceph-mon[67226]: daemon mds.foofs.smithi177.hqctzz is now active in filesystem foofs as rank 0 2024-04-12T21:57:49.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:49 smithi177 ceph-mon[52869]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/426 objects degraded (3.756%), 12 pgs degraded) 2024-04-12T21:57:49.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:49 smithi177 ceph-mon[52869]: mds.? [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] up:rejoin 2024-04-12T21:57:49.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:49 smithi177 ceph-mon[52869]: fsmap foofs:1/1 {0=foofs.smithi177.hqctzz=up:rejoin} 2024-04-12T21:57:49.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:49 smithi177 ceph-mon[52869]: daemon mds.foofs.smithi177.hqctzz is now active in filesystem foofs as rank 0 2024-04-12T21:57:50.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:50 smithi123 ceph-mon[67226]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-12T21:57:50.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:50 smithi123 ceph-mon[67226]: mds.? [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] up:active 2024-04-12T21:57:50.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:50 smithi123 ceph-mon[67226]: fsmap foofs:1 {0=foofs.smithi177.hqctzz=up:active} 2024-04-12T21:57:50.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:50 smithi123 ceph-mon[67226]: pgmap v323: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 341 B/s wr, 11 op/s; 412 KiB/s, 17 objects/s recovering 2024-04-12T21:57:50.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:50 smithi177 ceph-mon[52869]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-12T21:57:50.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:50 smithi177 ceph-mon[52869]: mds.? [v2:172.21.15.177:6832/2017305461,v1:172.21.15.177:6833/2017305461] up:active 2024-04-12T21:57:50.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:50 smithi177 ceph-mon[52869]: fsmap foofs:1 {0=foofs.smithi177.hqctzz=up:active} 2024-04-12T21:57:50.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:50 smithi177 ceph-mon[52869]: pgmap v323: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 341 B/s wr, 11 op/s; 412 KiB/s, 17 objects/s recovering 2024-04-12T21:57:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:52 smithi177 ceph-mon[52869]: pgmap v324: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 391 B/s wr, 10 op/s; 315 KiB/s, 13 objects/s recovering 2024-04-12T21:57:52.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:52 smithi123 ceph-mon[67226]: pgmap v324: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 391 B/s wr, 10 op/s; 315 KiB/s, 13 objects/s recovering 2024-04-12T21:57:54.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:54 smithi177 ceph-mon[52869]: pgmap v325: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 442 B/s wr, 11 op/s; 267 KiB/s, 11 objects/s recovering 2024-04-12T21:57:54.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:54 smithi123 ceph-mon[67226]: pgmap v325: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 442 B/s wr, 11 op/s; 267 KiB/s, 11 objects/s recovering 2024-04-12T21:57:55.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:55.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:55.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:57:55.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:55 smithi123 ceph-mon[67226]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-12T21:57:55.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:55 smithi123 ceph-mon[67226]: mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678] up:boot 2024-04-12T21:57:55.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:55 smithi123 ceph-mon[67226]: fsmap foofs:1 {0=foofs.smithi177.hqctzz=up:active} 1 up:standby 2024-04-12T21:57:55.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:57:55.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:55 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:55.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:55.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:55.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:57:55.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:55 smithi177 ceph-mon[52869]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-12T21:57:55.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:55 smithi177 ceph-mon[52869]: mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678] up:boot 2024-04-12T21:57:55.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:55 smithi177 ceph-mon[52869]: fsmap foofs:1 {0=foofs.smithi177.hqctzz=up:active} 1 up:standby 2024-04-12T21:57:55.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:57:55.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:55 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:57:56.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:56 smithi177 ceph-mon[52869]: pgmap v326: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.1 KiB/s wr, 13 op/s; 227 KiB/s, 9 objects/s recovering 2024-04-12T21:57:56.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:56 smithi123 ceph-mon[67226]: pgmap v326: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.1 KiB/s wr, 13 op/s; 227 KiB/s, 9 objects/s recovering 2024-04-12T21:57:59.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:57:58 smithi177 ceph-mon[52869]: pgmap v327: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 12 op/s; 0 B/s, 7 objects/s recovering 2024-04-12T21:57:59.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:57:58 smithi123 ceph-mon[67226]: pgmap v327: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 12 op/s; 0 B/s, 7 objects/s recovering 2024-04-12T21:57:59.878 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:58:00.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:00.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:00.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:00 smithi123 ceph-mon[67226]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:00.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:00 smithi123 ceph-mon[67226]: pgmap v328: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-04-12T21:58:00.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:00.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:58:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:00.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:00.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:00 smithi177 ceph-mon[52869]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:00.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:00 smithi177 ceph-mon[52869]: pgmap v328: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-04-12T21:58:00.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:00.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (9m) 1s ago 19m 24.7M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (6m) 1s ago 19m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (6m) 25s ago 18m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 1s ago 18m - - 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (6s) 1s ago 13m 11.2M - 19.0.0-2053-gb277feb2 668895944f0e b60128d4fa3b 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (13m) 25s ago 13m 15.8M - 17.2.0 e1d6a67b021e bc05443d8d83 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (8m) 1s ago 20m 514M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (8m) 25s ago 18m 453M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (7m) 1s ago 20m 59.8M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (7m) 25s ago 18m 45.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:58:00.568 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (12m) 1s ago 12m 274M - 4.0 e1d6a67b021e a1fcf6a99454 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (12m) 25s ago 12m 87.5M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (9m) 1s ago 18m 19.4M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (9m) 25s ago 18m 17.0M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (5m) 1s ago 17m 392M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (4m) 1s ago 17m 449M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (3m) 1s ago 16m 408M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (3m) 1s ago 16m 411M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (2m) 25s ago 15m 416M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (106s) 25s ago 15m 308M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (69s) 25s ago 15m 297M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7ce736a7262b 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (29s) 25s ago 14m 12.2M 2704M 19.0.0-2053-gb277feb2 668895944f0e 5bea7c19fe20 2024-04-12T21:58:00.569 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (8m) 1s ago 17m 71.5M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:58:00.999 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:58:00.999 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:58:00.999 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:58:00.999 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 13 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:58:01.000 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:58:01.344 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "osd" 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "15/23 daemons upgraded", 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mds daemons", 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:58:01.345 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:58:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:01 smithi177 ceph-mon[52869]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:01 smithi177 ceph-mon[52869]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:01 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3585359467' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:01.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:01 smithi123 ceph-mon[67226]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:01.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:01.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:01.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:01 smithi123 ceph-mon[67226]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:01.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:01 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3585359467' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:01.773 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:58:01.773 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:58:01.773 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:58:02.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:02 smithi123 ceph-mon[67226]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:02.401 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:02 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3802252177' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:58:02.401 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:02 smithi123 ceph-mon[67226]: pgmap v329: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-04-12T21:58:02.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:02 smithi177 ceph-mon[52869]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:02.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:02 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3802252177' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:58:02.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:02 smithi177 ceph-mon[52869]: pgmap v329: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-04-12T21:58:05.194 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: pgmap v330: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.1 KiB/s wr, 5 op/s 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:05.195 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:05.196 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi177.hqctzz"]}]: dispatch 2024-04-12T21:58:05.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: pgmap v330: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.1 KiB/s wr, 5 op/s 2024-04-12T21:58:05.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:05.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi177.hqctzz"]}]: dispatch 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: Detected new or changed devices on smithi123 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: pgmap v331: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 500 B/s rd, 8.2 KiB/s wr, 4 op/s 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: pgmap v332: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 248 B/s rd, 0 op/s 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: Upgrade: It appears safe to stop mds.foofs.smithi177.hqctzz 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: Upgrade: Updating mds.foofs.smithi177.hqctzz 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi177.hqctzz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:06.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:05 smithi177 ceph-mon[52869]: Deploying daemon mds.foofs.smithi177.hqctzz on smithi177 2024-04-12T21:58:06.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: Detected new or changed devices on smithi123 2024-04-12T21:58:06.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: pgmap v331: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 500 B/s rd, 8.2 KiB/s wr, 4 op/s 2024-04-12T21:58:06.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: pgmap v332: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 248 B/s rd, 0 op/s 2024-04-12T21:58:06.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: Upgrade: It appears safe to stop mds.foofs.smithi177.hqctzz 2024-04-12T21:58:06.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: Upgrade: Updating mds.foofs.smithi177.hqctzz 2024-04-12T21:58:06.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:06.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi177.hqctzz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T21:58:06.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:06.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:05 smithi123 ceph-mon[67226]: Deploying daemon mds.foofs.smithi177.hqctzz on smithi177 2024-04-12T21:58:07.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:06 smithi177 ceph-mon[52869]: pgmap v333: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 248 B/s wr, 0 op/s 2024-04-12T21:58:07.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:06 smithi123 ceph-mon[67226]: pgmap v333: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 248 B/s wr, 0 op/s 2024-04-12T21:58:08.169 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T21:58:07.908+0000 7f0fd4bdf700 -1 bdev(0x555b2e6e2000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 3 2024-04-12T21:58:08.169 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T21:58:07.908+0000 7f0fd33dc700 -1 bdev(0x555b2e6e2000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 6 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[67221]: 2024-04-12T21:58:07.812+0000 7f13a5559700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-mon[67226]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-mon[67226]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-mon[67226]: osdmap e159: 8 total, 8 up, 8 in 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-mon[67226]: Standby daemon mds.foofs.smithi123.poxexz assigned to filesystem foofs as rank 0 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-mon[67226]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-mon[67226]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-mon[67226]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-12T21:58:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:07 smithi123 ceph-mon[67226]: fsmap foofs:1/1 {0=foofs.smithi123.poxexz=up:replay} 2024-04-12T21:58:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:07 smithi177 ceph-mon[52869]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-12T21:58:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:07 smithi177 ceph-mon[52869]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T21:58:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:07 smithi177 ceph-mon[52869]: osdmap e159: 8 total, 8 up, 8 in 2024-04-12T21:58:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:07 smithi177 ceph-mon[52869]: Standby daemon mds.foofs.smithi123.poxexz assigned to filesystem foofs as rank 0 2024-04-12T21:58:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:07 smithi177 ceph-mon[52869]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-12T21:58:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:07 smithi177 ceph-mon[52869]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T21:58:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:07 smithi177 ceph-mon[52869]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-12T21:58:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:07 smithi177 ceph-mon[52869]: fsmap foofs:1/1 {0=foofs.smithi123.poxexz=up:replay} 2024-04-12T21:58:09.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:08 smithi177 ceph-mon[52869]: pgmap v335: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 328 B/s rd, 492 B/s wr, 0 op/s 2024-04-12T21:58:09.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:08 smithi123 ceph-mon[67226]: pgmap v335: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 328 B/s rd, 492 B/s wr, 0 op/s 2024-04-12T21:58:10.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:09 smithi177 ceph-mon[52869]: reconnect by client.14548 172.21.15.123:0/3378876472 after 0.000999975 2024-04-12T21:58:10.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:09 smithi177 ceph-mon[52869]: reconnect by client.24323 172.21.15.177:0/3783509255 after 0.000999975 2024-04-12T21:58:10.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:09 smithi177 ceph-mon[52869]: mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678] up:reconnect 2024-04-12T21:58:10.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:09 smithi177 ceph-mon[52869]: fsmap foofs:1/1 {0=foofs.smithi123.poxexz=up:reconnect} 2024-04-12T21:58:10.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:09 smithi123 ceph-mon[67226]: reconnect by client.14548 172.21.15.123:0/3378876472 after 0.000999975 2024-04-12T21:58:10.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:09 smithi123 ceph-mon[67226]: reconnect by client.24323 172.21.15.177:0/3783509255 after 0.000999975 2024-04-12T21:58:10.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:09 smithi123 ceph-mon[67226]: mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678] up:reconnect 2024-04-12T21:58:10.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:09 smithi123 ceph-mon[67226]: fsmap foofs:1/1 {0=foofs.smithi123.poxexz=up:reconnect} 2024-04-12T21:58:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:10 smithi177 ceph-mon[52869]: mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678] up:rejoin 2024-04-12T21:58:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:10 smithi177 ceph-mon[52869]: fsmap foofs:1/1 {0=foofs.smithi123.poxexz=up:rejoin} 2024-04-12T21:58:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:10 smithi177 ceph-mon[52869]: daemon mds.foofs.smithi123.poxexz is now active in filesystem foofs as rank 0 2024-04-12T21:58:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:10 smithi177 ceph-mon[52869]: pgmap v336: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 492 B/s wr, 2 op/s 2024-04-12T21:58:11.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:10 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:58:11.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:10 smithi123 ceph-mon[67226]: mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678] up:rejoin 2024-04-12T21:58:11.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:10 smithi123 ceph-mon[67226]: fsmap foofs:1/1 {0=foofs.smithi123.poxexz=up:rejoin} 2024-04-12T21:58:11.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:10 smithi123 ceph-mon[67226]: daemon mds.foofs.smithi123.poxexz is now active in filesystem foofs as rank 0 2024-04-12T21:58:11.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:10 smithi123 ceph-mon[67226]: pgmap v336: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 492 B/s wr, 2 op/s 2024-04-12T21:58:11.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:10 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:58:12.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:11 smithi177 ceph-mon[52869]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-12T21:58:12.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:11 smithi177 ceph-mon[52869]: mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678] up:active 2024-04-12T21:58:12.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:11 smithi177 ceph-mon[52869]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 2024-04-12T21:58:12.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:11 smithi123 ceph-mon[67226]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-12T21:58:12.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:11 smithi123 ceph-mon[67226]: mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678] up:active 2024-04-12T21:58:12.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:11 smithi123 ceph-mon[67226]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 2024-04-12T21:58:13.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:12 smithi177 ceph-mon[52869]: pgmap v337: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 639 B/s wr, 8 op/s 2024-04-12T21:58:13.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:12 smithi123 ceph-mon[67226]: pgmap v337: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 639 B/s wr, 8 op/s 2024-04-12T21:58:15.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:15 smithi123 ceph-mon[67226]: pgmap v338: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 511 B/s wr, 11 op/s 2024-04-12T21:58:15.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:15.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:58:15.525 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:15 smithi177 ceph-mon[52869]: pgmap v338: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 511 B/s wr, 11 op/s 2024-04-12T21:58:15.525 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:15.525 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:58:17.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:17.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:17.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:17.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:16 smithi123 ceph-mon[67226]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-12T21:58:17.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:16 smithi123 ceph-mon[67226]: mds.? [v2:172.21.15.177:6832/3123617932,v1:172.21.15.177:6833/3123617932] up:boot 2024-04-12T21:58:17.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:16 smithi123 ceph-mon[67226]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:58:17.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:16 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:58:17.209 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:17.209 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:17.209 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:17.209 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:16 smithi177 ceph-mon[52869]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-12T21:58:17.209 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:16 smithi177 ceph-mon[52869]: mds.? [v2:172.21.15.177:6832/3123617932,v1:172.21.15.177:6833/3123617932] up:boot 2024-04-12T21:58:17.209 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:16 smithi177 ceph-mon[52869]: fsmap foofs:1 {0=foofs.smithi123.poxexz=up:active} 1 up:standby 2024-04-12T21:58:17.209 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:16 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:58:18.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:17 smithi177 ceph-mon[52869]: pgmap v339: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 8.8 KiB/s wr, 14 op/s 2024-04-12T21:58:18.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:17 smithi123 ceph-mon[67226]: pgmap v339: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 8.8 KiB/s wr, 14 op/s 2024-04-12T21:58:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:19 smithi177 ceph-mon[52869]: pgmap v340: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 8.4 KiB/s wr, 14 op/s 2024-04-12T21:58:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:20.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:19 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:20.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:19 smithi123 ceph-mon[67226]: pgmap v340: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 8.4 KiB/s wr, 14 op/s 2024-04-12T21:58:20.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:20.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:19 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:21.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:21.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:21 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:21.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:21 smithi177 ceph-mon[52869]: pgmap v341: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-04-12T21:58:21.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:21.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:21 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:21.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:21 smithi123 ceph-mon[67226]: pgmap v341: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-04-12T21:58:23.391 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:23 smithi177 ceph-mon[52869]: pgmap v342: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-04-12T21:58:23.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:23 smithi123 ceph-mon[67226]: pgmap v342: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: Detected new or changed devices on smithi177 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: pgmap v343: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 7.5 KiB/s wr, 6 op/s 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: pgmap v344: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 220 B/s rd, 9.1 KiB/s wr, 4 op/s 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.621 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all mds 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi123.poxexz"}]': finished 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi177.hqctzz"}]': finished 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all rgw 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all rbd-mirror 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.622 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all ceph-exporter 2024-04-12T21:58:24.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all iscsi 2024-04-12T21:58:24.623 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:24 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: Detected new or changed devices on smithi177 2024-04-12T21:58:24.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: pgmap v343: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 7.5 KiB/s wr, 6 op/s 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: pgmap v344: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 220 B/s rd, 9.1 KiB/s wr, 4 op/s 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all mds 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi123.poxexz"}]: dispatch 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi123.poxexz"}]': finished 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi177.hqctzz"}]: dispatch 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi177.hqctzz"}]': finished 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all rgw 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all rbd-mirror 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all ceph-exporter 2024-04-12T21:58:24.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:24.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:24.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all iscsi 2024-04-12T21:58:24.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:24 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:26.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: pgmap v345: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: Upgrade: Updating nfs.foo.0.0.smithi123.cfxrdw (1/2) 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: Creating key for client.nfs.foo.0.0.smithi123.cfxrdw 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T21:58:26.670 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: Rados config object exists: conf-nfs.foo 2024-04-12T21:58:26.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: Creating key for client.nfs.foo.0.0.smithi123.cfxrdw-rgw 2024-04-12T21:58:26.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T21:58:26.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: Bind address in nfs.foo.0.0.smithi123.cfxrdw's ganesha conf is defaulting to empty 2024-04-12T21:58:26.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:26.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:26 smithi123 ceph-mon[67226]: Deploying daemon nfs.foo.0.0.smithi123.cfxrdw on smithi123 2024-04-12T21:58:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: pgmap v345: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 220 B/s rd, 0 op/s 2024-04-12T21:58:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: Upgrade: Updating nfs.foo.0.0.smithi123.cfxrdw (1/2) 2024-04-12T21:58:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: Creating key for client.nfs.foo.0.0.smithi123.cfxrdw 2024-04-12T21:58:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T21:58:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-12T21:58:26.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: Rados config object exists: conf-nfs.foo 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: Creating key for client.nfs.foo.0.0.smithi123.cfxrdw-rgw 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.cfxrdw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: Bind address in nfs.foo.0.0.smithi123.cfxrdw's ganesha conf is defaulting to empty 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:26.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:26 smithi177 ceph-mon[52869]: Deploying daemon nfs.foo.0.0.smithi123.cfxrdw on smithi123 2024-04-12T21:58:28.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:28 smithi177 ceph-mon[52869]: pgmap v346: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 220 B/s wr, 0 op/s 2024-04-12T21:58:28.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:28 smithi123 ceph-mon[67226]: pgmap v346: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 220 B/s wr, 0 op/s 2024-04-12T21:58:29.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:29 smithi123 ceph-mon[67226]: from='mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678]' entity='mds.foofs.smithi123.poxexz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.123:0/3378876472"}]: dispatch 2024-04-12T21:58:29.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:29 smithi177 ceph-mon[52869]: from='mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678]' entity='mds.foofs.smithi123.poxexz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.123:0/3378876472"}]: dispatch 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: Evicting (and blocklisting) client session 14548 (172.21.15.123:0/3378876472) 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: pgmap v347: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 551 B/s wr, 1 op/s 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: from='mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678]' entity='mds.foofs.smithi123.poxexz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.123:0/3378876472"}]': finished 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: osdmap e160: 8 total, 8 up, 8 in 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:58:30.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.8"}]: dispatch 2024-04-12T21:58:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: Evicting (and blocklisting) client session 14548 (172.21.15.123:0/3378876472) 2024-04-12T21:58:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: pgmap v347: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 551 B/s wr, 1 op/s 2024-04-12T21:58:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: from='mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678]' entity='mds.foofs.smithi123.poxexz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.123:0/3378876472"}]': finished 2024-04-12T21:58:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: osdmap e160: 8 total, 8 up, 8 in 2024-04-12T21:58:30.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:30.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:30.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:30.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:58:30.920 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.8"}]: dispatch 2024-04-12T21:58:32.133 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:58:32.142 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.8"}]': finished 2024-04-12T21:58:32.142 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: osdmap e161: 8 total, 8 up, 8 in 2024-04-12T21:58:32.142 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: Upgrade: Updating nfs.foo.1.0.smithi177.vvnnsx (2/2) 2024-04-12T21:58:32.142 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: Creating key for client.nfs.foo.1.0.smithi177.vvnnsx 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: Rados config object exists: conf-nfs.foo 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: Creating key for client.nfs.foo.1.0.smithi177.vvnnsx-rgw 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: Bind address in nfs.foo.1.0.smithi177.vvnnsx's ganesha conf is defaulting to empty 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:32.143 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: Deploying daemon nfs.foo.1.0.smithi177.vvnnsx on smithi177 2024-04-12T21:58:32.144 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:32 smithi177 ceph-mon[52869]: pgmap v350: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-12T21:58:32.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.8"}]': finished 2024-04-12T21:58:32.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: osdmap e161: 8 total, 8 up, 8 in 2024-04-12T21:58:32.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: Upgrade: Updating nfs.foo.1.0.smithi177.vvnnsx (2/2) 2024-04-12T21:58:32.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:32.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: Creating key for client.nfs.foo.1.0.smithi177.vvnnsx 2024-04-12T21:58:32.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T21:58:32.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-12T21:58:32.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: Rados config object exists: conf-nfs.foo 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: Creating key for client.nfs.foo.1.0.smithi177.vvnnsx-rgw 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi177.vvnnsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: Bind address in nfs.foo.1.0.smithi177.vvnnsx's ganesha conf is defaulting to empty 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: Deploying daemon nfs.foo.1.0.smithi177.vvnnsx on smithi177 2024-04-12T21:58:32.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:32 smithi123 ceph-mon[67226]: pgmap v350: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-12T21:58:32.797 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:58:32.797 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (10m) 33s ago 19m 24.7M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:58:32.797 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (6m) 33s ago 19m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:58:32.797 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (6m) 13s ago 19m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 33s ago 19m - - 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (38s) 33s ago 13m 11.2M - 19.0.0-2053-gb277feb2 668895944f0e b60128d4fa3b 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (17s) 13s ago 13m 12.3M - 19.0.0-2053-gb277feb2 668895944f0e fb7edd2196fe 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (9m) 33s ago 20m 514M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (8m) 13s ago 18m 453M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (8m) 33s ago 21m 59.8M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (7m) 13s ago 18m 46.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 starting - - - - 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (13m) 13s ago 13m 87.6M - 4.0 e1d6a67b021e 4b9fb9111054 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (10m) 33s ago 19m 19.4M - 1.3.1 1dbe0e931976 7924de44c067 2024-04-12T21:58:32.798 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (9m) 13s ago 18m 17.4M - 1.3.1 1dbe0e931976 ce9ea5f93b07 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (6m) 33s ago 18m 392M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (5m) 33s ago 17m 449M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (4m) 33s ago 17m 408M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (3m) 33s ago 16m 411M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (2m) 13s ago 16m 419M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (2m) 13s ago 16m 312M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (101s) 13s ago 15m 299M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7ce736a7262b 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (62s) 13s ago 15m 244M 2704M 19.0.0-2053-gb277feb2 668895944f0e 5bea7c19fe20 2024-04-12T21:58:32.799 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (8m) 33s ago 18m 71.5M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:58:33.214 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:58:33.214 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:58:33.215 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:58:33.360 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:33 smithi177 ceph-mon[52869]: osdmap e162: 8 total, 8 up, 8 in 2024-04-12T21:58:33.361 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:33 smithi177 ceph-mon[52869]: from='client.34524 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:33.361 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:33 smithi177 ceph-mon[52869]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:33.361 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:33 smithi177 ceph-mon[52869]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:33.367 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:33 smithi123 ceph-mon[67226]: osdmap e162: 8 total, 8 up, 8 in 2024-04-12T21:58:33.367 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:33 smithi123 ceph-mon[67226]: from='client.34524 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:33.368 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:33 smithi123 ceph-mon[67226]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:33.368 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:33 smithi123 ceph-mon[67226]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:33.609 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "osd", 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "mds" 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "16/23 daemons upgraded", 2024-04-12T21:58:33.610 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading nfs daemons", 2024-04-12T21:58:33.611 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:58:33.611 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:58:34.111 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:58:34.111 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:58:34.111 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:58:34.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:34 smithi177 ceph-mon[52869]: osdmap e163: 8 total, 8 up, 8 in 2024-04-12T21:58:34.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:34 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1891581168' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:34.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:34 smithi177 ceph-mon[52869]: pgmap v353: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-12T21:58:34.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:34 smithi177 ceph-mon[52869]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:34.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:34 smithi123 ceph-mon[67226]: osdmap e163: 8 total, 8 up, 8 in 2024-04-12T21:58:34.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:34 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1891581168' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:34.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:34 smithi123 ceph-mon[67226]: pgmap v353: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-12T21:58:34.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:34 smithi123 ceph-mon[67226]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:58:35.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:35 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3872541275' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:58:35.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:35 smithi123 ceph-mon[67226]: osdmap e164: 8 total, 8 up, 8 in 2024-04-12T21:58:35.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:35 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:35.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:35 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:35.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:35 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:35.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:35 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3872541275' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:58:35.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:35 smithi177 ceph-mon[52869]: osdmap e164: 8 total, 8 up, 8 in 2024-04-12T21:58:35.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:35 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:35.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:35 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:35.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:35 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:36.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:36 smithi123 ceph-mon[67226]: pgmap v355: 97 pgs: 1 active+recovering+undersized+remapped, 96 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 701 B/s wr, 5 op/s; 0 B/s, 78 objects/s recovering 2024-04-12T21:58:36.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:36 smithi123 ceph-mon[67226]: osdmap e165: 8 total, 8 up, 8 in 2024-04-12T21:58:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:36 smithi177 ceph-mon[52869]: pgmap v355: 97 pgs: 1 active+recovering+undersized+remapped, 96 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 701 B/s wr, 5 op/s; 0 B/s, 78 objects/s recovering 2024-04-12T21:58:37.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:36 smithi177 ceph-mon[52869]: osdmap e165: 8 total, 8 up, 8 in 2024-04-12T21:58:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:38 smithi177 ceph-mon[52869]: pgmap v357: 97 pgs: 1 active+recovering+undersized+remapped, 96 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 762 B/s wr, 4 op/s; 0 B/s, 64 objects/s recovering 2024-04-12T21:58:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:38 smithi177 ceph-mon[52869]: Evicting (and blocklisting) client session 24323 (172.21.15.177:0/3783509255) 2024-04-12T21:58:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:38 smithi177 ceph-mon[52869]: from='mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678]' entity='mds.foofs.smithi123.poxexz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.177:0/3783509255"}]: dispatch 2024-04-12T21:58:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:38 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:39.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:38 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:39.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:38 smithi123 ceph-mon[67226]: pgmap v357: 97 pgs: 1 active+recovering+undersized+remapped, 96 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 762 B/s wr, 4 op/s; 0 B/s, 64 objects/s recovering 2024-04-12T21:58:39.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:38 smithi123 ceph-mon[67226]: Evicting (and blocklisting) client session 24323 (172.21.15.177:0/3783509255) 2024-04-12T21:58:39.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:38 smithi123 ceph-mon[67226]: from='mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678]' entity='mds.foofs.smithi123.poxexz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.177:0/3783509255"}]: dispatch 2024-04-12T21:58:39.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:38 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:39.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:38 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:39 smithi123 ceph-mon[67226]: from='mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678]' entity='mds.foofs.smithi123.poxexz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.177:0/3783509255"}]': finished 2024-04-12T21:58:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:39 smithi123 ceph-mon[67226]: osdmap e166: 8 total, 8 up, 8 in 2024-04-12T21:58:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:39.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:39 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:39.995 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:39 smithi177 ceph-mon[52869]: from='mds.? [v2:172.21.15.123:6834/3464607678,v1:172.21.15.123:6835/3464607678]' entity='mds.foofs.smithi123.poxexz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.177:0/3783509255"}]': finished 2024-04-12T21:58:39.995 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:39 smithi177 ceph-mon[52869]: osdmap e166: 8 total, 8 up, 8 in 2024-04-12T21:58:39.995 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:39.995 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:39 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:40.983 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:40 smithi177 ceph-mon[52869]: pgmap v359: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.5 KiB/s wr, 6 op/s; 0 B/s, 121 objects/s recovering 2024-04-12T21:58:40.983 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:40.984 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:40 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:41.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:40 smithi123 ceph-mon[67226]: pgmap v359: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.5 KiB/s wr, 6 op/s; 0 B/s, 121 objects/s recovering 2024-04-12T21:58:41.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:41.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:40 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:42.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:42.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:42.264 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:42.264 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:42 smithi177 ceph-mon[52869]: pgmap v360: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.9 KiB/s wr, 6 op/s; 0 B/s, 57 objects/s recovering 2024-04-12T21:58:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:43.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:43.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:42 smithi123 ceph-mon[67226]: pgmap v360: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.9 KiB/s wr, 6 op/s; 0 B/s, 57 objects/s recovering 2024-04-12T21:58:43.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:43.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: pgmap v361: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.6 KiB/s wr, 5 op/s; 0 B/s, 48 objects/s recovering 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi123.cfxrdw"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi123.cfxrdw"}]': finished 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi177.vvnnsx"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi177.vvnnsx"}]': finished 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:44 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: pgmap v361: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.6 KiB/s wr, 5 op/s; 0 B/s, 48 objects/s recovering 2024-04-12T21:58:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:58:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:58:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi123.cfxrdw"}]: dispatch 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi123.cfxrdw"}]': finished 2024-04-12T21:58:45.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi177.vvnnsx"}]: dispatch 2024-04-12T21:58:45.284 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi177.vvnnsx"}]': finished 2024-04-12T21:58:45.284 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:58:45.284 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:44 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:46.077 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:45 smithi177 ceph-mon[52869]: pgmap v362: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.5 KiB/s wr, 5 op/s; 0 B/s, 43 objects/s recovering 2024-04-12T21:58:46.077 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:45 smithi177 ceph-mon[52869]: pgmap v363: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.7 KiB/s wr, 5 op/s; 0 B/s, 54 objects/s recovering 2024-04-12T21:58:46.077 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:45 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all nfs 2024-04-12T21:58:46.078 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:45 smithi177 ceph-mon[52869]: Upgrade: Setting container_image for all nvmeof 2024-04-12T21:58:46.078 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:58:46.078 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:46.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:45 smithi123 ceph-mon[67226]: pgmap v362: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.5 KiB/s wr, 5 op/s; 0 B/s, 43 objects/s recovering 2024-04-12T21:58:46.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:45 smithi123 ceph-mon[67226]: pgmap v363: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.7 KiB/s wr, 5 op/s; 0 B/s, 54 objects/s recovering 2024-04-12T21:58:46.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:45 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all nfs 2024-04-12T21:58:46.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:45 smithi123 ceph-mon[67226]: Upgrade: Setting container_image for all nvmeof 2024-04-12T21:58:46.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:58:46.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:48.062 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:47 smithi123 ceph-mon[67226]: Upgrade: Updating node-exporter.smithi123 (1/2) 2024-04-12T21:58:48.062 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:47 smithi123 ceph-mon[67226]: Deploying daemon node-exporter.smithi123 on smithi123 2024-04-12T21:58:48.062 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:47 smithi123 ceph-mon[67226]: pgmap v364: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 130 B/s rd, 521 B/s wr, 0 op/s 2024-04-12T21:58:48.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:47 smithi177 ceph-mon[52869]: Upgrade: Updating node-exporter.smithi123 (1/2) 2024-04-12T21:58:48.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:47 smithi177 ceph-mon[52869]: Deploying daemon node-exporter.smithi123 on smithi123 2024-04-12T21:58:48.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:47 smithi177 ceph-mon[52869]: pgmap v364: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 130 B/s rd, 521 B/s wr, 0 op/s 2024-04-12T21:58:50.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:49 smithi123 ceph-mon[67226]: pgmap v365: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 225 B/s rd, 450 B/s wr, 0 op/s 2024-04-12T21:58:50.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:49 smithi177 ceph-mon[52869]: pgmap v365: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 225 B/s rd, 450 B/s wr, 0 op/s 2024-04-12T21:58:51.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:51 smithi123 ceph-mon[67226]: pgmap v366: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 225 B/s rd, 0 B/s wr, 0 op/s 2024-04-12T21:58:51.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:51 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:51.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:51 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:52.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:51 smithi177 ceph-mon[52869]: pgmap v366: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 225 B/s rd, 0 B/s wr, 0 op/s 2024-04-12T21:58:52.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:51 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:52.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:51 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:54.130 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:53 smithi177 ceph-mon[52869]: pgmap v367: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 112 B/s rd, 0 op/s 2024-04-12T21:58:54.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:53 smithi123 ceph-mon[67226]: pgmap v367: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 112 B/s rd, 0 op/s 2024-04-12T21:58:55.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:54 smithi177 ceph-mon[52869]: Upgrade: Updating node-exporter.smithi177 (2/2) 2024-04-12T21:58:55.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:54 smithi177 ceph-mon[52869]: Deploying daemon node-exporter.smithi177 on smithi177 2024-04-12T21:58:55.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:54 smithi123 ceph-mon[67226]: Upgrade: Updating node-exporter.smithi177 (2/2) 2024-04-12T21:58:55.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:54 smithi123 ceph-mon[67226]: Deploying daemon node-exporter.smithi177 on smithi177 2024-04-12T21:58:56.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:55 smithi123 ceph-mon[67226]: pgmap v368: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T21:58:56.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:55 smithi177 ceph-mon[52869]: pgmap v368: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T21:58:57.979 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:57 smithi177 ceph-mon[52869]: pgmap v369: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:58:58.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:57 smithi123 ceph-mon[67226]: pgmap v369: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:58:59.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:59 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:59.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:59 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:59.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:59 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:59.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:58:59 smithi177 ceph-mon[52869]: pgmap v370: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:58:59.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:59 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:59.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:59 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:58:59.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:59 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:58:59.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:58:59 smithi123 ceph-mon[67226]: pgmap v370: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:00.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:59:00.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:59:01.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:01 smithi177 ceph-mon[52869]: pgmap v371: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:01.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:01.532 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:01 smithi123 ceph-mon[67226]: pgmap v371: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:01.532 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:01.532 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:03.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:03.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:03.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:03 smithi123 ceph-mon[67226]: pgmap v372: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:03.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:03.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:03 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:03.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:03 smithi177 ceph-mon[52869]: pgmap v372: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:03.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:03 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:04.483 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:59:05.150 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (11m) 2s ago 20m 24.7M - 0.23.0 ba2b418f427c c0c0e3a30cbf 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (7m) 2s ago 20m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (7m) 4s ago 19m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 2s ago 19m - - 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (71s) 2s ago 14m 35.7M - 19.0.0-2053-gb277feb2 668895944f0e b60128d4fa3b 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (49s) 4s ago 14m 12.5M - 19.0.0-2053-gb277feb2 668895944f0e fb7edd2196fe 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (9m) 2s ago 21m 516M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (9m) 4s ago 19m 454M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (8m) 2s ago 21m 62.9M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (8m) 4s ago 19m 49.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (36s) 2s ago 13m 60.0M - 5.7 668895944f0e 3dddd8f63059 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (30s) 4s ago 13m 63.7M - 5.7 668895944f0e 323d26cf7e22 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (13s) 2s ago 19m 8056k - 1.5.0 0da6a335fe13 024f2399ce77 2024-04-12T21:59:05.151 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (7s) 4s ago 19m 3590k - 1.5.0 0da6a335fe13 7dad6a43acd2 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (6m) 2s ago 18m 394M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (5m) 2s ago 18m 480M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (5m) 2s ago 17m 410M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (4m) 2s ago 17m 424M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (3m) 4s ago 17m 417M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (2m) 4s ago 16m 311M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (2m) 4s ago 16m 299M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7ce736a7262b 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (94s) 4s ago 15m 244M 2704M 19.0.0-2053-gb277feb2 668895944f0e 5bea7c19fe20 2024-04-12T21:59:05.152 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (9m) 2s ago 19m 71.5M - 2.33.4 514e6a882f6e 866f42e854fb 2024-04-12T21:59:05.152 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:05.152 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:04 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:05.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:05.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:04 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:05.550 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:59:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:59:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:59:05.550 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:59:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:59:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:59:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:59:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:59:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T21:59:05.551 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:59:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:59:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:59:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:59:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:59:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T21:59:05.552 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:59:05.553 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "nfs", 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "osd", 2024-04-12T21:59:05.889 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-04-12T21:59:05.890 INFO:teuthology.orchestra.run.smithi123.stdout: "mds" 2024-04-12T21:59:05.890 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:59:05.890 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "18/23 daemons upgraded", 2024-04-12T21:59:05.890 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-12T21:59:05.890 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:59:05.890 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:59:06.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:05 smithi177 ceph-mon[52869]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:06.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:05 smithi177 ceph-mon[52869]: pgmap v373: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:06.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:05 smithi177 ceph-mon[52869]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:06.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:05 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3258649874' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:06.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:06.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:05 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:06.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:05 smithi123 ceph-mon[67226]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:06.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:05 smithi123 ceph-mon[67226]: pgmap v373: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:06.072 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:05 smithi123 ceph-mon[67226]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:06.072 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:05 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3258649874' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:06.072 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:06.072 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:05 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:06.293 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:59:06.294 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:59:06.294 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:59:07.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:06 smithi123 ceph-mon[67226]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:07.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:06 smithi123 ceph-mon[67226]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:07.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:06 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/603318232' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:59:07.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:06 smithi177 ceph-mon[52869]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:07.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:06 smithi177 ceph-mon[52869]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:07.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:06 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/603318232' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:59:08.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: pgmap v374: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:08.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:07 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: pgmap v374: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:08.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:07 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:09.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:08 smithi123 ceph-mon[67226]: pgmap v375: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 183 B/s rd, 0 op/s 2024-04-12T21:59:09.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:08 smithi123 ceph-mon[67226]: pgmap v376: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 111 B/s rd, 0 op/s 2024-04-12T21:59:09.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:08 smithi177 ceph-mon[52869]: pgmap v375: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 183 B/s rd, 0 op/s 2024-04-12T21:59:09.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:08 smithi177 ceph-mon[52869]: pgmap v376: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 111 B/s rd, 0 op/s 2024-04-12T21:59:11.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:10 smithi123 ceph-mon[67226]: Upgrade: Updating prometheus.smithi123 2024-04-12T21:59:11.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:10 smithi123 ceph-mon[67226]: Deploying daemon prometheus.smithi123 on smithi123 2024-04-12T21:59:11.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:10 smithi123 ceph-mon[67226]: pgmap v377: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-04-12T21:59:11.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:10 smithi177 ceph-mon[52869]: Upgrade: Updating prometheus.smithi123 2024-04-12T21:59:11.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:10 smithi177 ceph-mon[52869]: Deploying daemon prometheus.smithi123 on smithi123 2024-04-12T21:59:11.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:10 smithi177 ceph-mon[52869]: pgmap v377: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-04-12T21:59:13.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:12 smithi123 ceph-mon[67226]: pgmap v378: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-04-12T21:59:13.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:12 smithi177 ceph-mon[52869]: pgmap v378: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-04-12T21:59:15.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:14 smithi123 ceph-mon[67226]: pgmap v379: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-04-12T21:59:15.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:14 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:15.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:14 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:15.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:14 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:15.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:14 smithi177 ceph-mon[52869]: pgmap v379: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-04-12T21:59:15.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:14 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:15.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:14 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:15.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:14 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:16.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:59:16.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:16.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:16.246 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:59:16.246 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:16.246 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:17.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:16 smithi123 ceph-mon[67226]: pgmap v380: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-04-12T21:59:17.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:16 smithi177 ceph-mon[52869]: pgmap v380: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-04-12T21:59:19.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:18 smithi123 ceph-mon[67226]: pgmap v381: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T21:59:19.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:18 smithi177 ceph-mon[52869]: pgmap v381: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T21:59:21.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:20 smithi177 ceph-mon[52869]: pgmap v382: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:21.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:20 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:21.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:20 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:21.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:20 smithi123 ceph-mon[67226]: pgmap v382: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:21.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:20 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:21.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:20 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:22.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:22 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:22.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:22 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:22.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:22 smithi123 ceph-mon[67226]: pgmap v383: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:22.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:22 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:22.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:22 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:22.531 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:22 smithi177 ceph-mon[52869]: pgmap v383: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:24.833 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:24 smithi123 ceph-mon[67226]: pgmap v384: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:24 smithi177 ceph-mon[52869]: pgmap v384: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: pgmap v385: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T21:59:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: pgmap v386: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:26.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.032 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.033 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:25 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: pgmap v385: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: pgmap v386: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:26.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:26.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:25 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:27.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:26 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:59:27.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:26 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T21:59:28.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:27 smithi123 ceph-mon[67226]: Upgrade: Updating alertmanager.smithi123 2024-04-12T21:59:28.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:27 smithi123 ceph-mon[67226]: Deploying daemon alertmanager.smithi123 on smithi123 2024-04-12T21:59:28.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:27 smithi123 ceph-mon[67226]: pgmap v387: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:27 smithi177 ceph-mon[52869]: Upgrade: Updating alertmanager.smithi123 2024-04-12T21:59:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:27 smithi177 ceph-mon[52869]: Deploying daemon alertmanager.smithi123 on smithi123 2024-04-12T21:59:28.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:27 smithi177 ceph-mon[52869]: pgmap v387: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:30.118 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:29 smithi123 ceph-mon[67226]: pgmap v388: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:30.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:29 smithi177 ceph-mon[52869]: pgmap v388: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:31.096 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:59:31.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:59:32.079 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:31 smithi123 ceph-mon[67226]: pgmap v389: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:32.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:31 smithi177 ceph-mon[52869]: pgmap v389: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:33.593 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:33.593 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:33.593 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:33.594 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:33 smithi123 ceph-mon[67226]: pgmap v390: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:33.594 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:33.594 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:33 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:33.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:33.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:33.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:33.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:33 smithi177 ceph-mon[52869]: pgmap v390: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 222 B/s rd, 0 op/s 2024-04-12T21:59:33.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:33.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:33 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:36.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:35 smithi123 ceph-mon[67226]: pgmap v391: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T21:59:36.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:35 smithi177 ceph-mon[52869]: pgmap v391: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T21:59:36.667 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T21:59:37.331 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (5s) 0s ago 20m 16.3M - 0.25.0 c8568f914cd2 ccdbc0296781 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (7m) 0s ago 20m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (7m) 36s ago 20m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 error 0s ago 20m - - 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (103s) 0s ago 14m 35.9M - 19.0.0-2053-gb277feb2 668895944f0e b60128d4fa3b 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (81s) 36s ago 14m 12.5M - 19.0.0-2053-gb277feb2 668895944f0e fb7edd2196fe 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (10m) 0s ago 21m 519M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (9m) 36s ago 20m 454M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (9m) 0s ago 22m 63.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (8m) 36s ago 19m 49.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T21:59:37.332 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (68s) 0s ago 14m 60.3M - 5.7 668895944f0e 3dddd8f63059 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (62s) 36s ago 14m 63.7M - 5.7 668895944f0e 323d26cf7e22 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (46s) 0s ago 20m 8296k - 1.5.0 0da6a335fe13 024f2399ce77 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (39s) 36s ago 19m 3590k - 1.5.0 0da6a335fe13 7dad6a43acd2 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (7m) 0s ago 19m 395M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (6m) 0s ago 18m 479M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (5m) 0s ago 18m 411M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (4m) 0s ago 17m 425M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (4m) 36s ago 17m 417M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (3m) 36s ago 17m 311M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (2m) 36s ago 16m 299M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7ce736a7262b 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (2m) 36s ago 16m 244M 2704M 19.0.0-2053-gb277feb2 668895944f0e 5bea7c19fe20 2024-04-12T21:59:37.333 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (22s) 0s ago 19m 50.8M - 2.43.0 a07b618ecd1d 9b3bb576d598 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T21:59:37.735 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:59:37.736 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T21:59:37.736 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T21:59:37.736 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T21:59:37.736 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T21:59:37.736 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T21:59:37.736 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T21:59:37.736 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:59:38.062 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:37 smithi123 ceph-mon[67226]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:38.063 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:37 smithi123 ceph-mon[67226]: pgmap v392: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:38.063 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:38.063 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:37 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:38.063 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:37 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/2070318217' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:38.076 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T21:59:38.076 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T21:59:38.076 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-04-12T21:59:38.076 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T21:59:38.077 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-04-12T21:59:38.077 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-04-12T21:59:38.077 INFO:teuthology.orchestra.run.smithi123.stdout: "nfs", 2024-04-12T21:59:38.077 INFO:teuthology.orchestra.run.smithi123.stdout: "mon", 2024-04-12T21:59:38.077 INFO:teuthology.orchestra.run.smithi123.stdout: "osd", 2024-04-12T21:59:38.077 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-04-12T21:59:38.077 INFO:teuthology.orchestra.run.smithi123.stdout: "mds" 2024-04-12T21:59:38.078 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T21:59:38.078 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "18/23 daemons upgraded", 2024-04-12T21:59:38.078 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading alertmanager daemons", 2024-04-12T21:59:38.078 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T21:59:38.078 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T21:59:38.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:37 smithi177 ceph-mon[52869]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:38.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:37 smithi177 ceph-mon[52869]: pgmap v392: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:38.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:38.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:37 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:38.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:37 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/2070318217' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:38.492 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T21:59:38.492 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T21:59:38.493 INFO:teuthology.orchestra.run.smithi123.stdout: daemon grafana.smithi123 on smithi123 is in error state 2024-04-12T21:59:39.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:38 smithi123 ceph-mon[67226]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:39.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:38 smithi123 ceph-mon[67226]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:39.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:38 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:39.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:38 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:39.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:38 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1874230275' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:59:39.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:38 smithi177 ceph-mon[52869]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:39.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:38 smithi177 ceph-mon[52869]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:39.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:38 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:39.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:38 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:39.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:38 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1874230275' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T21:59:40.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:39 smithi123 ceph-mon[67226]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:40.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:39 smithi123 ceph-mon[67226]: pgmap v393: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:40.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:39 smithi177 ceph-mon[52869]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T21:59:40.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:39 smithi177 ceph-mon[52869]: pgmap v393: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:42.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:41 smithi123 ceph-mon[67226]: pgmap v394: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:42.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:42.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:42.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:42.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:42.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:42.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:41 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:41 smithi177 ceph-mon[52869]: pgmap v394: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:42.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:41 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:43.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: pgmap v395: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 0 op/s 2024-04-12T21:59:43.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: pgmap v396: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 114 B/s rd, 0 op/s 2024-04-12T21:59:43.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.170 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:42 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: pgmap v395: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 186 B/s rd, 0 op/s 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: pgmap v396: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 114 B/s rd, 0 op/s 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.282 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:43.283 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:42 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:44.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:43 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:59:44.320 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:43 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T21:59:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:44 smithi123 ceph-mon[67226]: Upgrade: Updating grafana.smithi123 2024-04-12T21:59:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:44 smithi123 ceph-mon[67226]: Deploying daemon grafana.smithi123 on smithi123 2024-04-12T21:59:45.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:44 smithi123 ceph-mon[67226]: pgmap v397: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 0 op/s 2024-04-12T21:59:45.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:44 smithi177 ceph-mon[52869]: Upgrade: Updating grafana.smithi123 2024-04-12T21:59:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:44 smithi177 ceph-mon[52869]: Deploying daemon grafana.smithi123 on smithi123 2024-04-12T21:59:45.281 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:44 smithi177 ceph-mon[52869]: pgmap v397: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 0 op/s 2024-04-12T21:59:46.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:59:46.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T21:59:47.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:46 smithi177 ceph-mon[52869]: pgmap v398: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 0 op/s 2024-04-12T21:59:47.303 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:46 smithi123 ceph-mon[67226]: pgmap v398: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 0 op/s 2024-04-12T21:59:48.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:48 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:48.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:48 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:48.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:48 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:48.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:48 smithi123 ceph-mon[67226]: pgmap v399: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 114 B/s rd, 0 op/s 2024-04-12T21:59:48.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:48 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:48.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:48 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:48.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:48 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:48.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:48 smithi177 ceph-mon[52869]: pgmap v399: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 114 B/s rd, 0 op/s 2024-04-12T21:59:51.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:50 smithi177 ceph-mon[52869]: pgmap v400: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 0 op/s 2024-04-12T21:59:51.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:50 smithi123 ceph-mon[67226]: pgmap v400: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 0 op/s 2024-04-12T21:59:53.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:52 smithi177 ceph-mon[52869]: pgmap v401: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T21:59:53.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:53.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:52 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:53.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:52 smithi123 ceph-mon[67226]: pgmap v401: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T21:59:53.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:53.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:52 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:54.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:54 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:54.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:54 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:54.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:54 smithi123 ceph-mon[67226]: pgmap v402: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:54 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:54 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:54 smithi177 ceph-mon[52869]: pgmap v402: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T21:59:57.249 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:56 smithi123 ceph-mon[67226]: pgmap v403: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:57.280 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:56 smithi177 ceph-mon[52869]: pgmap v403: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: pgmap v404: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:59:58.420 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: Upgrade: Finalizing container_image settings 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-12T21:59:58.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-12T21:59:58.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: Upgrade: Complete! 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:58.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:58.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:58.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:58.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:58.781 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: pgmap v404: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.782 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: Upgrade: Finalizing container_image settings 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-12T21:59:58.783 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.784 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: Upgrade: Complete! 2024-04-12T21:59:58.785 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:58.786 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T21:59:59.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:59 smithi123 ceph-mon[67226]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T21:59:59.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 21:59:59 smithi123 ceph-mon[67226]: Cluster is now healthy 2024-04-12T21:59:59.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:59 smithi177 ceph-mon[52869]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T21:59:59.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 21:59:59 smithi177 ceph-mon[52869]: Cluster is now healthy 2024-04-12T22:00:00.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:00 smithi123 ceph-mon[67226]: pgmap v405: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 268 B/s rd, 0 op/s 2024-04-12T22:00:00.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:00 smithi123 ceph-mon[67226]: overall HEALTH_OK 2024-04-12T22:00:00.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T22:00:00.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:00:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:00 smithi177 ceph-mon[52869]: pgmap v405: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 268 B/s rd, 0 op/s 2024-04-12T22:00:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:00 smithi177 ceph-mon[52869]: overall HEALTH_OK 2024-04-12T22:00:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T22:00:00.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:00:01.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:01 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T22:00:01.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:01 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T22:00:02.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:02 smithi177 ceph-mon[52869]: pgmap v406: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-12T22:00:02.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:02 smithi123 ceph-mon[67226]: pgmap v406: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-12T22:00:04.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:04 smithi177 ceph-mon[52869]: pgmap v407: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 268 B/s rd, 0 op/s 2024-04-12T22:00:04.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:04 smithi123 ceph-mon[67226]: pgmap v407: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 268 B/s rd, 0 op/s 2024-04-12T22:00:06.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:06 smithi177 ceph-mon[52869]: pgmap v408: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-12T22:00:06.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:06 smithi123 ceph-mon[67226]: pgmap v408: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-12T22:00:08.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:08 smithi177 ceph-mon[52869]: pgmap v409: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-12T22:00:08.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:08 smithi123 ceph-mon[67226]: pgmap v409: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 179 B/s rd, 0 op/s 2024-04-12T22:00:09.528 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T22:00:09.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:09 smithi177 ceph-mon[52869]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:00:09.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:09 smithi123 ceph-mon[67226]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:00:10.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:10 smithi177 ceph-mon[52869]: pgmap v410: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:10.826 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:10 smithi123 ceph-mon[67226]: pgmap v410: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (39s) 18s ago 21m 16.3M - 0.25.0 c8568f914cd2 ccdbc0296781 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (8m) 18s ago 21m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (8m) 70s ago 20m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (24s) 18s ago 20m 48.8M - 9.4.12 2bacad6d85d8 a44b46352470 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (2m) 18s ago 15m 36.0M - 19.0.0-2053-gb277feb2 668895944f0e b60128d4fa3b 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (116s) 70s ago 15m 12.5M - 19.0.0-2053-gb277feb2 668895944f0e fb7edd2196fe 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (11m) 18s ago 22m 525M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (10m) 70s ago 20m 454M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T22:00:11.543 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (9m) 18s ago 22m 63.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (9m) 70s ago 20m 49.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (102s) 18s ago 15m 60.5M - 5.7 668895944f0e 3dddd8f63059 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (97s) 70s ago 14m 63.7M - 5.7 668895944f0e 323d26cf7e22 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (80s) 18s ago 20m 8430k - 1.5.0 0da6a335fe13 024f2399ce77 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (73s) 70s ago 20m 3590k - 1.5.0 0da6a335fe13 7dad6a43acd2 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (7m) 18s ago 19m 395M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (6m) 18s ago 19m 480M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (6m) 18s ago 18m 411M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (5m) 18s ago 18m 425M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (4m) 70s ago 18m 417M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (3m) 70s ago 17m 311M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (3m) 70s ago 17m 299M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7ce736a7262b 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (2m) 70s ago 16m 244M 2704M 19.0.0-2053-gb277feb2 668895944f0e 5bea7c19fe20 2024-04-12T22:00:11.544 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (57s) 18s ago 20m 51.0M - 2.43.0 a07b618ecd1d 9b3bb576d598 2024-04-12T22:00:12.274 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions' 2024-04-12T22:00:12.581 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:12 smithi123 ceph-mon[67226]: pgmap v411: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:12.581 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:12 smithi123 ceph-mon[67226]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:00:12.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:12 smithi177 ceph-mon[52869]: pgmap v411: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:12.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:12 smithi177 ceph-mon[52869]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T22:00:14.245 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T22:00:14.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:00:14.246 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T22:00:14.246 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T22:00:14.246 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:00:14.246 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T22:00:14.246 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T22:00:14.246 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T22:00:14.246 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T22:00:14.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:14 smithi123 ceph-mon[67226]: pgmap v412: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:14.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:14 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3203083380' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T22:00:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:14 smithi177 ceph-mon[52869]: pgmap v412: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:14.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:14 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3203083380' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T22:00:14.867 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-12T22:00:15.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:00:15.916 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:00:16.502 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:16 smithi123 ceph-mon[67226]: pgmap v413: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:16.702 INFO:teuthology.orchestra.run.smithi123.stdout:wait for servicemap items w/ changing names to refresh 2024-04-12T22:00:16.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:16 smithi177 ceph-mon[52869]: pgmap v413: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:17.378 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'sleep 60' 2024-04-12T22:00:18.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:18 smithi123 ceph-mon[67226]: pgmap v414: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:18.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:18 smithi177 ceph-mon[52869]: pgmap v414: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:20.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:20 smithi177 ceph-mon[52869]: pgmap v415: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:20.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:20 smithi123 ceph-mon[67226]: pgmap v415: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:22.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:22 smithi177 ceph-mon[52869]: pgmap v416: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:22.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:22 smithi123 ceph-mon[67226]: pgmap v416: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:24.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:24 smithi177 ceph-mon[52869]: pgmap v417: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:24.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:24 smithi123 ceph-mon[67226]: pgmap v417: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:26.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:26 smithi177 ceph-mon[52869]: pgmap v418: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:26.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:26 smithi123 ceph-mon[67226]: pgmap v418: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:28.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:28 smithi177 ceph-mon[52869]: pgmap v419: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:28.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:28 smithi123 ceph-mon[67226]: pgmap v419: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:30.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:30 smithi177 ceph-mon[52869]: pgmap v420: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:30.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:00:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:30 smithi123 ceph-mon[67226]: pgmap v420: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:30.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:00:32.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:32 smithi177 ceph-mon[52869]: pgmap v421: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:32.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:32 smithi123 ceph-mon[67226]: pgmap v421: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:34.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:34 smithi177 ceph-mon[52869]: pgmap v422: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:34.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:34 smithi123 ceph-mon[67226]: pgmap v422: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:36.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:36 smithi177 ceph-mon[52869]: pgmap v423: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:36.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:36 smithi123 ceph-mon[67226]: pgmap v423: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:38.780 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:38 smithi177 ceph-mon[52869]: pgmap v424: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:38.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:38 smithi123 ceph-mon[67226]: pgmap v424: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:40.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:40 smithi123 ceph-mon[67226]: pgmap v425: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:41.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:40 smithi177 ceph-mon[52869]: pgmap v425: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:42.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:42 smithi123 ceph-mon[67226]: pgmap v426: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:42 smithi177 ceph-mon[52869]: pgmap v426: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:44.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:44 smithi123 ceph-mon[67226]: pgmap v427: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:45.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:44 smithi177 ceph-mon[52869]: pgmap v427: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:45.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:00:45.931 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:00:46.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:46 smithi123 ceph-mon[67226]: pgmap v428: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:47.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:46 smithi177 ceph-mon[52869]: pgmap v428: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:48.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:48 smithi123 ceph-mon[67226]: pgmap v429: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:49.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:48 smithi177 ceph-mon[52869]: pgmap v429: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:50.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:50 smithi123 ceph-mon[67226]: pgmap v430: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:51.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:50 smithi177 ceph-mon[52869]: pgmap v430: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:52.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:52 smithi123 ceph-mon[67226]: pgmap v431: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:53.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:52 smithi177 ceph-mon[52869]: pgmap v431: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:54.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:54 smithi123 ceph-mon[67226]: pgmap v432: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:55.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:54 smithi177 ceph-mon[52869]: pgmap v432: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:00:56.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:56 smithi123 ceph-mon[67226]: pgmap v433: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:57.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:56 smithi177 ceph-mon[52869]: pgmap v433: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:58.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:58 smithi123 ceph-mon[67226]: pgmap v434: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:58.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:00:58.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:00:58.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:00:58.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T22:00:58.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:00:58 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T22:00:59.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:58 smithi177 ceph-mon[52869]: pgmap v434: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:00:59.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:00:59.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:00:59.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:00:59.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T22:00:59.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:00:58 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' 2024-04-12T22:01:00.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:00 smithi123 ceph-mon[67226]: pgmap v435: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:00.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:00 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:01:01.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:00 smithi177 ceph-mon[52869]: pgmap v435: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:01.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:00 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:01:02.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:02 smithi123 ceph-mon[67226]: pgmap v436: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:03.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:02 smithi177 ceph-mon[52869]: pgmap v436: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:04.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:04 smithi123 ceph-mon[67226]: pgmap v437: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:05.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:04 smithi177 ceph-mon[52869]: pgmap v437: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:06.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:06 smithi123 ceph-mon[67226]: pgmap v438: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:07.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:06 smithi177 ceph-mon[52869]: pgmap v438: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:08.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:08 smithi123 ceph-mon[67226]: pgmap v439: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:09.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:08 smithi177 ceph-mon[52869]: pgmap v439: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:10.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:10 smithi123 ceph-mon[67226]: pgmap v440: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:11.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:10 smithi177 ceph-mon[52869]: pgmap v440: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:12.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:12 smithi123 ceph-mon[67226]: pgmap v441: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:13.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:12 smithi177 ceph-mon[52869]: pgmap v441: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:14.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:14 smithi123 ceph-mon[67226]: pgmap v442: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:15.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:14 smithi177 ceph-mon[52869]: pgmap v442: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:15.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:15 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:01:15.931 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:15 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:01:16.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:16 smithi123 ceph-mon[67226]: pgmap v443: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:17.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:16 smithi177 ceph-mon[52869]: pgmap v443: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:18.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:18 smithi123 ceph-mon[67226]: pgmap v444: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:19.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:18 smithi177 ceph-mon[52869]: pgmap v444: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:19.720 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T22:01:20.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:20 smithi123 ceph-mon[67226]: pgmap v445: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:21.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:20 smithi177 ceph-mon[52869]: pgmap v445: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (110s) 89s ago 22m 16.3M - 0.25.0 c8568f914cd2 ccdbc0296781 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (9m) 89s ago 22m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 33ca5107d74d 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi177 smithi177 running (9m) 2m ago 21m 6665k - 19.0.0-2053-gb277feb2 668895944f0e ca14ccf2bb1f 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (94s) 89s ago 22m 48.8M - 9.4.12 2bacad6d85d8 a44b46352470 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.poxexz smithi123 running (3m) 89s ago 16m 36.0M - 19.0.0-2053-gb277feb2 668895944f0e b60128d4fa3b 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi177.hqctzz smithi177 running (3m) 2m ago 16m 12.5M - 19.0.0-2053-gb277feb2 668895944f0e fb7edd2196fe 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.psjrfs smithi123 *:8443,9283,8765 running (12m) 89s ago 23m 525M - 19.0.0-2053-gb277feb2 668895944f0e b4e2577c4ac3 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi177.whybza smithi177 *:8443,9283,8765 running (11m) 2m ago 21m 454M - 19.0.0-2053-gb277feb2 668895944f0e 2ec33f67482a 2024-04-12T22:01:21.761 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (11m) 89s ago 23m 63.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e ef4cffbb2b46 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi177 smithi177 running (10m) 2m ago 21m 49.1M 2048M 19.0.0-2053-gb277feb2 668895944f0e c090c7cd3f2e 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.cfxrdw smithi123 *:2049 running (2m) 89s ago 16m 60.5M - 5.7 668895944f0e 3dddd8f63059 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi177.vvnnsx smithi177 *:2049 running (2m) 2m ago 16m 63.7M - 5.7 668895944f0e 323d26cf7e22 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (2m) 89s ago 22m 8430k - 1.5.0 0da6a335fe13 024f2399ce77 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi177 smithi177 *:9100 running (2m) 2m ago 21m 3590k - 1.5.0 0da6a335fe13 7dad6a43acd2 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (8m) 89s ago 21m 395M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8555f56fa001 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (8m) 89s ago 20m 480M 1936M 19.0.0-2053-gb277feb2 668895944f0e 5308afb6272c 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (7m) 89s ago 20m 411M 1936M 19.0.0-2053-gb277feb2 668895944f0e 013228f4e63d 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (6m) 89s ago 19m 425M 1936M 19.0.0-2053-gb277feb2 668895944f0e 7ca563e272ac 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi177 running (5m) 2m ago 19m 417M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7dafde6ecf2c 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi177 running (5m) 2m ago 18m 311M 2704M 19.0.0-2053-gb277feb2 668895944f0e 49cfbac67f9f 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi177 running (4m) 2m ago 18m 299M 2704M 19.0.0-2053-gb277feb2 668895944f0e 7ce736a7262b 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi177 running (3m) 2m ago 18m 244M 2704M 19.0.0-2053-gb277feb2 668895944f0e 5bea7c19fe20 2024-04-12T22:01:21.762 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (2m) 89s ago 21m 51.0M - 2.43.0 a07b618ecd1d 9b3bb576d598 2024-04-12T22:01:22.448 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade status' 2024-04-12T22:01:22.754 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:22 smithi123 ceph-mon[67226]: pgmap v446: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:22.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:22 smithi123 ceph-mon[67226]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:01:23.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:22 smithi177 ceph-mon[52869]: pgmap v446: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:23.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:22 smithi177 ceph-mon[52869]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": null, 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": false, 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "", 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": null, 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-04-12T22:01:24.363 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T22:01:24.668 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:24 smithi123 ceph-mon[67226]: pgmap v447: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:24.958 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph health detail' 2024-04-12T22:01:25.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:24 smithi177 ceph-mon[52869]: pgmap v447: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:25.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:25 smithi123 ceph-mon[67226]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:01:26.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:25 smithi177 ceph-mon[52869]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:01:26.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:26 smithi123 ceph-mon[67226]: pgmap v448: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:27.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:26 smithi177 ceph-mon[52869]: pgmap v448: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:27.031 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-04-12T22:01:27.662 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:27 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/3452659574' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T22:01:27.663 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions' 2024-04-12T22:01:28.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:27 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/3452659574' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T22:01:28.906 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:28 smithi123 ceph-mon[67226]: pgmap v449: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:29.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:28 smithi177 ceph-mon[52869]: pgmap v449: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:29.659 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-04-12T22:01:29.659 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-04-12T22:01:29.659 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T22:01:29.659 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:01:29.659 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-04-12T22:01:29.659 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T22:01:29.660 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-04-12T22:01:30.272 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-12T22:01:30.909 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:30 smithi123 ceph-mon[67226]: pgmap v450: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:30.909 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:30 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1661106770' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T22:01:30.909 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:30 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:01:31.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:30 smithi177 ceph-mon[52869]: pgmap v450: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:31.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:30 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1661106770' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T22:01:31.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:30 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:01:32.341 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-04-12T22:01:32.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:32 smithi123 ceph-mon[67226]: pgmap v451: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:32.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:32 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/2871328151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T22:01:32.973 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-12T22:01:33.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:32 smithi177 ceph-mon[52869]: pgmap v451: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:33.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:32 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/2871328151' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T22:01:34.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:34 smithi123 ceph-mon[67226]: pgmap v452: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:34.982 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)" 2024-04-12T22:01:35.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:34 smithi177 ceph-mon[52869]: pgmap v452: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:35.629 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-04-12T22:01:35.630 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-04-12T22:01:35.630 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T22:01:35.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:35 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/1908286920' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T22:01:36.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:35 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/1908286920' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T22:01:36.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:36 smithi123 ceph-mon[67226]: pgmap v453: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:37.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:36 smithi177 ceph-mon[52869]: pgmap v453: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:37.674 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-04-12T22:01:37.674 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T21:38:21.360615Z", "last_refresh": "2024-04-12T21:59:52.603620Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T21:38:16.121443Z", "last_refresh": "2024-04-12T21:59:01.081513Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-04-12T21:38:18.846014Z", "last_refresh": "2024-04-12T21:59:52.603785Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T21:44:41.585607Z", "last_refresh": "2024-04-12T21:59:01.082125Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T21:38:14.887058Z", "last_refresh": "2024-04-12T21:59:01.081612Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi177:172.21.15.177=smithi177"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T21:39:02.747478Z", "last_refresh": "2024-04-12T21:59:01.081702Z", "running": 2, "size": 2}}, {"events": ["2024-04-12T22:00:58.319540Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-04-12T21:45:02.605080Z", "last_refresh": "2024-04-12T21:59:01.082195Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T21:38:20.083904Z", "last_refresh": "2024-04-12T21:59:01.081775Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", "last_refresh": "2024-04-12T21:59:01.081846Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T21:38:17.660968Z", "last_refresh": "2024-04-12T21:59:52.603931Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T22:01:38.328 INFO:tasks.cephadm:nfs.foo has 2/2 2024-04-12T22:01:38.328 DEBUG:teuthology.parallel:result is None 2024-04-12T22:01:38.328 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-12T22:01:38.339 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T22:01:38.340 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-04-12T22:01:38.369 INFO:teuthology.orchestra.run.smithi123.stderr:+ umount /mnt/foo 2024-04-12T22:01:38.381 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T22:01:38.390 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T22:01:38.390 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-04-12T22:01:38.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:38 smithi123 ceph-mon[67226]: pgmap v454: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:38.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:38 smithi123 ceph-mon[67226]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T22:01:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:38 smithi177 ceph-mon[52869]: pgmap v454: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:39.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:38 smithi177 ceph-mon[52869]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T22:01:40.447 INFO:teuthology.orchestra.run.smithi123.stdout: "foo" 2024-04-12T22:01:40.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:40 smithi123 ceph-mon[67226]: pgmap v455: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:41.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:40 smithi177 ceph-mon[52869]: pgmap v455: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:41.108 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-04-12T22:01:41.665 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:41 smithi123 ceph-mon[67226]: from='client.34648 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:01:42.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:41 smithi177 ceph-mon[52869]: from='client.34648 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:01:42.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:42 smithi123 ceph-mon[67226]: pgmap v456: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:43.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:42 smithi177 ceph-mon[52869]: pgmap v456: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:43.142 INFO:teuthology.orchestra.run.smithi123.stdout:[ 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: { 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "access_type": "RW", 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "clients": [], 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "cluster_id": "foo", 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "export_id": 1, 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "fsal": { 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "fs_name": "foofs", 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "name": "CEPH", 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "user_id": "nfs.foo.1" 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "path": "/", 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: "protocols": [ 2024-04-12T22:01:43.143 INFO:teuthology.orchestra.run.smithi123.stdout: 4 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout: "pseudo": "/fake", 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout: "security_label": true, 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout: "squash": "none", 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout: "transports": [ 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout: "TCP" 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-04-12T22:01:43.144 INFO:teuthology.orchestra.run.smithi123.stdout:] 2024-04-12T22:01:43.806 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-04-12T22:01:44.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:44 smithi123 ceph-mon[67226]: from='client.34652 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:01:44.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:44 smithi123 ceph-mon[67226]: pgmap v457: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:45.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:44 smithi177 ceph-mon[52869]: from='client.34652 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:01:45.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:44 smithi177 ceph-mon[52869]: pgmap v457: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:45.583 INFO:teuthology.orchestra.run.smithi123.stdout:foo rec-0000000000000004:nfs.foo.1 2024-04-12T22:01:45.583 INFO:teuthology.orchestra.run.smithi123.stdout:foo grace 2024-04-12T22:01:45.583 INFO:teuthology.orchestra.run.smithi123.stdout:foo rec-0000000000000004:nfs.foo.0 2024-04-12T22:01:45.583 INFO:teuthology.orchestra.run.smithi123.stdout:foo conf-nfs.foo 2024-04-12T22:01:45.584 INFO:teuthology.orchestra.run.smithi123.stdout:foo export-1 2024-04-12T22:01:45.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:45 smithi123 ceph-mon[67226]: mgrmap e43: smithi123.psjrfs(active, since 11m), standbys: smithi177.whybza 2024-04-12T22:01:45.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:45 smithi123 ceph-mon[67226]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:01:45.931 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:45 smithi177 ceph-mon[52869]: mgrmap e43: smithi123.psjrfs(active, since 11m), standbys: smithi177.whybza 2024-04-12T22:01:45.931 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:45 smithi177 ceph-mon[52869]: from='mgr.34102 172.21.15.123:0/3541150195' entity='mgr.smithi123.psjrfs' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T22:01:46.291 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T22:01:46.301 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T22:01:46.301 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 -- bash -c 'set -ex 2024-04-12T22:01:46.301 DEBUG:teuthology.orchestra.run.smithi123:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-04-12T22:01:46.301 DEBUG:teuthology.orchestra.run.smithi123:> ' 2024-04-12T22:01:46.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:46 smithi123 ceph-mon[67226]: pgmap v458: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:47.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:46 smithi177 ceph-mon[52869]: pgmap v458: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T22:01:48.008 INFO:teuthology.orchestra.run.smithi123.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-04-12T22:01:48.242 INFO:teuthology.orchestra.run.smithi123.stderr:+ [[ 6 -gt 2 ]] 2024-04-12T22:01:48.827 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-12T22:01:48.838 INFO:tasks.cephadm:Teardown begin 2024-04-12T22:01:48.838 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T22:01:48.868 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T22:01:48.904 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-12T22:01:48.904 DEBUG:teuthology.orchestra.run.smithi123:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-12T22:01:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:48 smithi123 ceph-mon[67226]: pgmap v459: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:48.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:48 smithi123 ceph-mon[67226]: from='client.? 172.21.15.123:0/2074770225' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-12T22:01:48.923 DEBUG:teuthology.orchestra.run.smithi177:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-12T22:01:48.961 INFO:tasks.cephadm:Stopping all daemons... 2024-04-12T22:01:48.961 INFO:tasks.cephadm.mon.smithi123:Stopping mon.smithi123... 2024-04-12T22:01:48.962 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123 2024-04-12T22:01:49.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:48 smithi177 ceph-mon[52869]: pgmap v459: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T22:01:49.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:48 smithi177 ceph-mon[52869]: from='client.? 172.21.15.123:0/2074770225' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-12T22:01:49.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:49 smithi123 systemd[1]: Stopping Ceph mon.smithi123 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:01:49.617 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[67221]: 2024-04-12T22:01:49.267+0000 7f13aad64700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi123 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:01:49.617 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:49 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123[67221]: 2024-04-12T22:01:49.267+0000 7f13aad64700 -1 mon.smithi123@0(leader) e3 *** Got Signal Terminated *** 2024-04-12T22:01:49.617 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:49 smithi123 podman[113615]: 2024-04-12 22:01:49.297894596 +0000 UTC m=+0.152934665 container died ef4cffbb2b460eb2e393f735c33f4dc979238120664d0918f60d1d7ef52bb3ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, GIT_CLEAN=True, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-12T22:01:49.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:49 smithi123 podman[113615]: 2024-04-12 22:01:49.714960123 +0000 UTC m=+0.570000242 container cleanup ef4cffbb2b460eb2e393f735c33f4dc979238120664d0918f60d1d7ef52bb3ad (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-b277feb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T22:01:49.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Apr 12 22:01:49 smithi123 bash[113615]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi123 2024-04-12T22:01:50.287 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi123.service' 2024-04-12T22:01:50.340 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:01:50.341 INFO:tasks.cephadm.mon.smithi123:Stopped mon.smithi123 2024-04-12T22:01:50.341 INFO:tasks.cephadm.mon.smithi177:Stopping mon.smithi177... 2024-04-12T22:01:50.341 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi177 2024-04-12T22:01:50.651 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:50 smithi177 systemd[1]: Stopping Ceph mon.smithi177 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:01:51.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177[52865]: 2024-04-12T22:01:50.649+0000 7f90ad060700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi177 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:01:51.030 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177[52865]: 2024-04-12T22:01:50.649+0000 7f90ad060700 -1 mon.smithi177@1(peon) e3 *** Got Signal Terminated *** 2024-04-12T22:01:51.031 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:50 smithi177 podman[78773]: 2024-04-12 22:01:50.834175112 +0000 UTC m=+0.295527315 container died c090c7cd3f2e49bc65ed5296331b4c776d1022deb77bb5f02fc1352aaade08f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-12T22:01:51.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:51 smithi177 podman[78773]: 2024-04-12 22:01:51.342491177 +0000 UTC m=+0.803843425 container cleanup c090c7cd3f2e49bc65ed5296331b4c776d1022deb77bb5f02fc1352aaade08f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD) 2024-04-12T22:01:51.530 INFO:journalctl@ceph.mon.smithi177.smithi177.stdout:Apr 12 22:01:51 smithi177 bash[78773]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-mon-smithi177 2024-04-12T22:01:51.712 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@mon.smithi177.service' 2024-04-12T22:01:51.756 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:01:51.756 INFO:tasks.cephadm.mon.smithi177:Stopped mon.smithi177 2024-04-12T22:01:51.757 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-12T22:01:51.757 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.0 2024-04-12T22:01:52.061 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:51 smithi123 systemd[1]: Stopping Ceph osd.0 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:01:52.419 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:52 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[73612]: 2024-04-12T22:01:52.059+0000 7f1c0fcf9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:01:52.419 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:52 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[73612]: 2024-04-12T22:01:52.059+0000 7f1c0fcf9700 -1 osd.0 166 *** Got signal Terminated *** 2024-04-12T22:01:52.419 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:52 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0[73612]: 2024-04-12T22:01:52.059+0000 7f1c0fcf9700 -1 osd.0 166 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T22:01:57.913 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:57 smithi123 podman[113749]: 2024-04-12 22:01:57.65006387 +0000 UTC m=+5.693694834 container died 8555f56fa001d0eeeed11b159e7ec1068fd5de3891e9c2dcca92c38c2dee4f60 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T22:01:58.169 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:58 smithi123 podman[113749]: 2024-04-12 22:01:58.010070425 +0000 UTC m=+6.053701397 container cleanup 8555f56fa001d0eeeed11b159e7ec1068fd5de3891e9c2dcca92c38c2dee4f60 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb) 2024-04-12T22:01:58.169 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:58 smithi123 bash[113749]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0 2024-04-12T22:01:58.632 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:58 smithi123 podman[113766]: 2024-04-12 22:01:58.311387871 +0000 UTC m=+0.659796669 container remove 8555f56fa001d0eeeed11b159e7ec1068fd5de3891e9c2dcca92c38c2dee4f60 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-12T22:01:58.918 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:58 smithi123 podman[113860]: 2024-04-12 22:01:58.535539427 +0000 UTC m=+0.018546769 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T22:01:58.919 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:58 smithi123 podman[113860]: 2024-04-12 22:01:58.67107065 +0000 UTC m=+0.154077981 container create e7c1f6a79de97fe7ce12710d68212a912711d542f7b2911119d0747f8c3c232c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-b277feb, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-12T22:01:59.266 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:58 smithi123 podman[113860]: 2024-04-12 22:01:58.966941714 +0000 UTC m=+0.449949051 container init e7c1f6a79de97fe7ce12710d68212a912711d542f7b2911119d0747f8c3c232c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T22:01:59.266 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:58 smithi123 podman[113860]: 2024-04-12 22:01:58.970113621 +0000 UTC m=+0.453120956 container start e7c1f6a79de97fe7ce12710d68212a912711d542f7b2911119d0747f8c3c232c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T22:01:59.266 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:59 smithi123 podman[113860]: 2024-04-12 22:01:59.082005968 +0000 UTC m=+0.565013300 container attach e7c1f6a79de97fe7ce12710d68212a912711d542f7b2911119d0747f8c3c232c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, ceph=True) 2024-04-12T22:01:59.578 INFO:journalctl@ceph.osd.0.smithi123.stdout:Apr 12 22:01:59 smithi123 podman[113860]: 2024-04-12 22:01:59.265592375 +0000 UTC m=+0.748599709 container died e7c1f6a79de97fe7ce12710d68212a912711d542f7b2911119d0747f8c3c232c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T22:01:59.872 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.0.service' 2024-04-12T22:01:59.913 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:01:59.914 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-12T22:01:59.914 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-12T22:01:59.914 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.1 2024-04-12T22:02:00.169 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:01:59 smithi123 systemd[1]: Stopping Ceph osd.1 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:02:00.669 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T22:02:00.264+0000 7f0fec613700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:02:00.669 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T22:02:00.264+0000 7f0fec613700 -1 osd.1 166 *** Got signal Terminated *** 2024-04-12T22:02:00.669 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:00 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1[76306]: 2024-04-12T22:02:00.264+0000 7f0fec613700 -1 osd.1 166 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T22:02:05.919 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:05 smithi123 podman[113967]: 2024-04-12 22:02:05.649179682 +0000 UTC m=+5.510673552 container died 5308afb6272c994bac44fe849d714c2c4229c116645d7c26b672221272906f29 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-12T22:02:06.419 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:05 smithi123 podman[113967]: 2024-04-12 22:02:05.972177141 +0000 UTC m=+5.833671012 container cleanup 5308afb6272c994bac44fe849d714c2c4229c116645d7c26b672221272906f29 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-12T22:02:06.419 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:05 smithi123 bash[113967]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1 2024-04-12T22:02:06.419 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:06 smithi123 podman[113988]: 2024-04-12 22:02:06.215054157 +0000 UTC m=+0.563482901 container remove 5308afb6272c994bac44fe849d714c2c4229c116645d7c26b672221272906f29 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T22:02:06.919 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:06 smithi123 podman[114082]: 2024-04-12 22:02:06.550642047 +0000 UTC m=+0.017841127 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T22:02:06.919 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:06 smithi123 podman[114082]: 2024-04-12 22:02:06.684942608 +0000 UTC m=+0.152141716 container create 8af7b4f3931b39d6dcf407dfb03fa81ee1a7b7afcd3eb4f6143b721e46d05326 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409) 2024-04-12T22:02:06.919 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:06 smithi123 podman[114082]: 2024-04-12 22:02:06.894199855 +0000 UTC m=+0.361398922 container init 8af7b4f3931b39d6dcf407dfb03fa81ee1a7b7afcd3eb4f6143b721e46d05326 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-12T22:02:06.919 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:06 smithi123 podman[114082]: 2024-04-12 22:02:06.897431929 +0000 UTC m=+0.364631010 container start 8af7b4f3931b39d6dcf407dfb03fa81ee1a7b7afcd3eb4f6143b721e46d05326 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-12T22:02:07.419 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:07 smithi123 podman[114082]: 2024-04-12 22:02:07.00171384 +0000 UTC m=+0.468912914 container attach 8af7b4f3931b39d6dcf407dfb03fa81ee1a7b7afcd3eb4f6143b721e46d05326 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_BRANCH=HEAD) 2024-04-12T22:02:07.419 INFO:journalctl@ceph.osd.1.smithi123.stdout:Apr 12 22:02:07 smithi123 podman[114082]: 2024-04-12 22:02:07.187789958 +0000 UTC m=+0.654989042 container died 8af7b4f3931b39d6dcf407dfb03fa81ee1a7b7afcd3eb4f6143b721e46d05326 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-1-deactivate, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-12T22:02:07.832 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.1.service' 2024-04-12T22:02:07.874 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:02:07.874 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-12T22:02:07.874 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-12T22:02:07.875 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.2 2024-04-12T22:02:08.169 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:07 smithi123 systemd[1]: Stopping Ceph osd.2 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:02:08.669 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:08 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[78850]: 2024-04-12T22:02:08.241+0000 7fdb449ef700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:02:08.669 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:08 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[78850]: 2024-04-12T22:02:08.241+0000 7fdb449ef700 -1 osd.2 166 *** Got signal Terminated *** 2024-04-12T22:02:08.669 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:08 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2[78850]: 2024-04-12T22:02:08.241+0000 7fdb449ef700 -1 osd.2 166 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T22:02:14.051 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:13 smithi123 podman[114221]: 2024-04-12 22:02:13.770498773 +0000 UTC m=+5.669530991 container died 013228f4e63d19e433bdfb1e8df28a3e49d5b06c3aa7a972fe847dc4e232f648 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T22:02:14.384 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:14 smithi123 podman[114221]: 2024-04-12 22:02:14.158072453 +0000 UTC m=+6.057104670 container cleanup 013228f4e63d19e433bdfb1e8df28a3e49d5b06c3aa7a972fe847dc4e232f648 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, RELEASE=squid-b277feb, org.label-schema.build-date=20240409) 2024-04-12T22:02:14.384 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:14 smithi123 bash[114221]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2 2024-04-12T22:02:14.668 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:14 smithi123 podman[114238]: 2024-04-12 22:02:14.38393304 +0000 UTC m=+0.616232409 container remove 013228f4e63d19e433bdfb1e8df28a3e49d5b06c3aa7a972fe847dc4e232f648 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-12T22:02:14.979 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:14 smithi123 podman[114330]: 2024-04-12 22:02:14.601027082 +0000 UTC m=+0.018513368 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T22:02:14.980 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:14 smithi123 podman[114330]: 2024-04-12 22:02:14.710270429 +0000 UTC m=+0.127756696 container create de909b871c5b231c324c2b187b91e13a6534378914f3580e85b7569c9368e0a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb) 2024-04-12T22:02:14.980 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:14 smithi123 podman[114330]: 2024-04-12 22:02:14.979284498 +0000 UTC m=+0.396770778 container init de909b871c5b231c324c2b187b91e13a6534378914f3580e85b7569c9368e0a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0) 2024-04-12T22:02:15.271 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:14 smithi123 podman[114330]: 2024-04-12 22:02:14.982755308 +0000 UTC m=+0.400241577 container start de909b871c5b231c324c2b187b91e13a6534378914f3580e85b7569c9368e0a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux ) 2024-04-12T22:02:15.271 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:15 smithi123 podman[114330]: 2024-04-12 22:02:15.054206164 +0000 UTC m=+0.471692432 container attach de909b871c5b231c324c2b187b91e13a6534378914f3580e85b7569c9368e0a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T22:02:15.557 INFO:journalctl@ceph.osd.2.smithi123.stdout:Apr 12 22:02:15 smithi123 podman[114330]: 2024-04-12 22:02:15.271008097 +0000 UTC m=+0.688494381 container died de909b871c5b231c324c2b187b91e13a6534378914f3580e85b7569c9368e0a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T22:02:15.840 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.2.service' 2024-04-12T22:02:15.882 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:02:15.882 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-12T22:02:15.882 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-12T22:02:15.882 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.3 2024-04-12T22:02:16.169 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:15 smithi123 systemd[1]: Stopping Ceph osd.3 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:02:16.669 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:16 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T22:02:16.234+0000 7f2b5b35f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:02:16.669 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:16 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T22:02:16.234+0000 7f2b5b35f700 -1 osd.3 166 *** Got signal Terminated *** 2024-04-12T22:02:16.669 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:16 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T22:02:16.234+0000 7f2b5b35f700 -1 osd.3 166 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T22:02:19.168 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:18 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T22:02:18.802+0000 7f2b5394b700 -1 osd.3 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:53.107275+0000 front 2024-04-12T22:01:53.107355+0000 (oldest deadline 2024-04-12T22:02:18.307847+0000) 2024-04-12T22:02:19.757 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:19 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:19.430+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:20.169 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:19 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T22:02:19.841+0000 7f2b5394b700 -1 osd.3 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:53.107275+0000 front 2024-04-12T22:01:53.107355+0000 (oldest deadline 2024-04-12T22:02:18.307847+0000) 2024-04-12T22:02:20.774 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:20 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:20.409+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:21.169 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:20 smithi123 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3[81584]: 2024-04-12T22:02:20.840+0000 7f2b5394b700 -1 osd.3 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:53.107275+0000 front 2024-04-12T22:01:53.107355+0000 (oldest deadline 2024-04-12T22:02:18.307847+0000) 2024-04-12T22:02:21.530 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:21 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:21.401+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:22.087 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:21 smithi123 podman[114437]: 2024-04-12 22:02:21.806939288 +0000 UTC m=+5.706702866 container died 7ca563e272ac33b3e59a55a216c99ee82eb4876489f1d24bcedef2655b2259a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-12T22:02:22.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:22 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:22.014+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:22.419 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:22 smithi123 podman[114437]: 2024-04-12 22:02:22.168539017 +0000 UTC m=+6.068302603 container cleanup 7ca563e272ac33b3e59a55a216c99ee82eb4876489f1d24bcedef2655b2259a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T22:02:22.419 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:22 smithi123 bash[114437]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3 2024-04-12T22:02:22.779 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:22 smithi123 podman[114457]: 2024-04-12 22:02:22.436245048 +0000 UTC m=+0.624999516 container remove 7ca563e272ac33b3e59a55a216c99ee82eb4876489f1d24bcedef2655b2259a7 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-12T22:02:22.780 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:22 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:22.385+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:23.090 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:22 smithi123 podman[114548]: 2024-04-12 22:02:22.778712445 +0000 UTC m=+0.109749223 container create b1d0bcc5be21b42526cfbbf95946dd28b6d66564b369646bdc087eb74040962b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T22:02:23.091 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:22 smithi123 podman[114548]: 2024-04-12 22:02:22.686997486 +0000 UTC m=+0.018034268 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T22:02:23.091 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:23 smithi123 podman[114548]: 2024-04-12 22:02:23.006204705 +0000 UTC m=+0.337241486 container init b1d0bcc5be21b42526cfbbf95946dd28b6d66564b369646bdc087eb74040962b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-12T22:02:23.091 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:23 smithi123 podman[114548]: 2024-04-12 22:02:23.009492134 +0000 UTC m=+0.340528913 container start b1d0bcc5be21b42526cfbbf95946dd28b6d66564b369646bdc087eb74040962b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-12T22:02:23.091 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:23 smithi123 podman[114548]: 2024-04-12 22:02:23.0893966 +0000 UTC m=+0.420433380 container attach b1d0bcc5be21b42526cfbbf95946dd28b6d66564b369646bdc087eb74040962b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-12T22:02:23.280 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:22 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:22.780+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:52.758951+0000 front 2024-04-12T22:01:52.759117+0000 (oldest deadline 2024-04-12T22:02:22.759358+0000) 2024-04-12T22:02:23.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:23 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:23.018+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:23.418 INFO:journalctl@ceph.osd.3.smithi123.stdout:Apr 12 22:02:23 smithi123 podman[114548]: 2024-04-12 22:02:23.309277225 +0000 UTC m=+0.640314005 container died b1d0bcc5be21b42526cfbbf95946dd28b6d66564b369646bdc087eb74040962b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T22:02:23.707 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:23 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:23.382+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:23.954 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.3.service' 2024-04-12T22:02:23.990 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:23 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:23.738+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:52.758951+0000 front 2024-04-12T22:01:52.759117+0000 (oldest deadline 2024-04-12T22:02:22.759358+0000) 2024-04-12T22:02:23.990 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:23 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:23.705+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:23.994 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:02:23.995 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-12T22:02:23.995 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-12T22:02:23.995 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.4 2024-04-12T22:02:24.280 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:24 smithi177 systemd[1]: Stopping Ceph osd.4 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:02:24.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:23 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:23.988+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:24.697 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:24.381+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:24.697 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:24.400+0000 7f7f8fe1d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:02:24.698 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:24.401+0000 7f7f8fe1d700 -1 osd.4 166 *** Got signal Terminated *** 2024-04-12T22:02:24.698 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:24.401+0000 7f7f8fe1d700 -1 osd.4 166 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T22:02:24.968 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:24.742+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:52.758951+0000 front 2024-04-12T22:01:52.759117+0000 (oldest deadline 2024-04-12T22:02:22.759358+0000) 2024-04-12T22:02:24.968 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:24.696+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:25.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:24 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:24.967+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:25.689 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:25 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:25.374+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:25.962 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:25 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:25.688+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:25.963 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:25 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:25.745+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:52.758951+0000 front 2024-04-12T22:01:52.759117+0000 (oldest deadline 2024-04-12T22:02:22.759358+0000) 2024-04-12T22:02:26.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:25 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:25.961+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:26.700 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:26 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:26.364+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:26.995 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:26 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:26.788+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:52.758951+0000 front 2024-04-12T22:01:52.759117+0000 (oldest deadline 2024-04-12T22:02:22.759358+0000) 2024-04-12T22:02:26.996 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:26 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:26.699+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:27.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:26 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:26.994+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:27.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:26 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:26.994+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:27.731 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:27 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:27.400+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:28.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:27 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:27.730+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:28.030 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:27 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:27.795+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:52.758951+0000 front 2024-04-12T22:01:52.759117+0000 (oldest deadline 2024-04-12T22:02:22.759358+0000) 2024-04-12T22:02:28.030 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:27 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:27.795+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.760036+0000 front 2024-04-12T22:02:02.760078+0000 (oldest deadline 2024-04-12T22:02:26.859976+0000) 2024-04-12T22:02:28.449 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:28.034+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:28.449 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:28.034+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:28.739 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:28.447+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:28.739 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:28.447+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:28.999 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:28.737+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:28.999 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:28.737+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:28.999 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:28.753+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:52.758951+0000 front 2024-04-12T22:01:52.759117+0000 (oldest deadline 2024-04-12T22:02:22.759358+0000) 2024-04-12T22:02:28.999 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4[57802]: 2024-04-12T22:02:28.753+0000 7f7f88409700 -1 osd.4 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.760036+0000 front 2024-04-12T22:02:02.760078+0000 (oldest deadline 2024-04-12T22:02:26.859976+0000) 2024-04-12T22:02:29.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:28.997+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:29.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:28 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:28.997+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:29.717 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:29.420+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:29.718 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:29.420+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:29.997 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:29 smithi177 podman[78939]: 2024-04-12 22:02:29.808334554 +0000 UTC m=+5.610924678 container died 7dafde6ecf2c0a9191e4244d05b04cde026c882709b48f13de44ec7461effa47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-12T22:02:29.998 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:29.716+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:29.998 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:29.716+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:30.251 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:29.996+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:30.251 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:29 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:29.996+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:30.530 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:30 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:30.469+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:30.530 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:30 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:30.469+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:30.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:30 smithi177 podman[78939]: 2024-04-12 22:02:30.251279087 +0000 UTC m=+6.053869213 container cleanup 7dafde6ecf2c0a9191e4244d05b04cde026c882709b48f13de44ec7461effa47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T22:02:30.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:30 smithi177 bash[78939]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4 2024-04-12T22:02:30.531 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:30 smithi177 podman[78990]: 2024-04-12 22:02:30.444308767 +0000 UTC m=+0.630320388 container remove 7dafde6ecf2c0a9191e4244d05b04cde026c882709b48f13de44ec7461effa47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409) 2024-04-12T22:02:30.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:30 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:30.715+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:30.967 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:30 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:30.715+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:30.967 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:30 smithi177 podman[79084]: 2024-04-12 22:02:30.861883008 +0000 UTC m=+0.111290334 container create 8b18d3e2bcd80974c3914437fcc0c91c5dc1b45dc5dd81fa37ea3b62387c0935 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-12T22:02:30.967 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:30 smithi177 podman[79084]: 2024-04-12 22:02:30.768409539 +0000 UTC m=+0.017816830 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T22:02:31.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:31 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:31.041+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:31.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:31 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:31.041+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:31.280 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:31 smithi177 podman[79084]: 2024-04-12 22:02:31.064441163 +0000 UTC m=+0.313848444 container init 8b18d3e2bcd80974c3914437fcc0c91c5dc1b45dc5dd81fa37ea3b62387c0935 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T22:02:31.280 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:31 smithi177 podman[79084]: 2024-04-12 22:02:31.067843318 +0000 UTC m=+0.317250603 container start 8b18d3e2bcd80974c3914437fcc0c91c5dc1b45dc5dd81fa37ea3b62387c0935 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True) 2024-04-12T22:02:31.280 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:31 smithi177 podman[79084]: 2024-04-12 22:02:31.147594097 +0000 UTC m=+0.397001400 container attach 8b18d3e2bcd80974c3914437fcc0c91c5dc1b45dc5dd81fa37ea3b62387c0935 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409) 2024-04-12T22:02:31.726 INFO:journalctl@ceph.osd.4.smithi177.stdout:Apr 12 22:02:31 smithi177 podman[79084]: 2024-04-12 22:02:31.378558451 +0000 UTC m=+0.627965731 container died 8b18d3e2bcd80974c3914437fcc0c91c5dc1b45dc5dd81fa37ea3b62387c0935 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b277feb) 2024-04-12T22:02:31.727 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:31 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:31.449+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:31.727 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:31 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:31.449+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:32.001 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:31 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:31.734+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:32.002 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:31 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:31.734+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:32.025 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.4.service' 2024-04-12T22:02:32.066 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:02:32.067 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-12T22:02:32.067 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-12T22:02:32.067 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.5 2024-04-12T22:02:32.280 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:32 smithi177 systemd[1]: Stopping Ceph osd.5 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:02:32.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:32.086+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:32.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:32.086+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:32.711 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:32.404+0000 7f19d45bd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:02:32.711 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:32.404+0000 7f19d45bd700 -1 osd.5 166 *** Got signal Terminated *** 2024-04-12T22:02:32.711 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:32.404+0000 7f19d45bd700 -1 osd.5 166 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T22:02:32.711 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:32.461+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:32.712 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:32.461+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:33.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:32.710+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:33.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:32 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:32.710+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:33.489 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:33.074+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:33.489 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:33.074+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:33.745 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:33.487+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:33.745 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:33.487+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:34.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:33.744+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:34.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:33 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:33.744+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:34.443 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:34 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:34.048+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:34.443 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:34 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:34.048+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:34.727 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:34 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:34.441+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:34.727 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:34 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:34.441+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:35.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:34 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:34.726+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:35.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:34 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:34.726+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:35.461 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:35 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:35.056+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:35.461 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:35 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:35.056+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:35.721 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:35 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:35.459+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:35.721 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:35 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:35.459+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:36.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:35 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:35.719+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:36.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:35 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:35.719+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:36.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:35 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:35.719+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:36.448 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:36.030+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:36.448 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:36.030+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:36.769 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:36.446+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.661627+0000 front 2024-04-12T22:01:54.661598+0000 (oldest deadline 2024-04-12T22:02:18.761516+0000) 2024-04-12T22:02:36.769 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5[60070]: 2024-04-12T22:02:36.446+0000 7f19d0bb1700 -1 osd.5 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:02.262603+0000 front 2024-04-12T22:02:02.262643+0000 (oldest deadline 2024-04-12T22:02:27.562495+0000) 2024-04-12T22:02:37.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:36.768+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:37.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:36.768+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:37.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:36.768+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:37.030 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:36.983+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:37.031 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:36.983+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:37.031 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:36 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:36.983+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:38.197 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:37 smithi177 podman[79196]: 2024-04-12 22:02:37.898725182 +0000 UTC m=+5.605458991 container died 49cfbac67f9fb7d54c94b1595521838dc7e96c32acfb7f2a0724c51bb0de63ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-b277feb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409) 2024-04-12T22:02:38.198 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:37 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:37.781+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:38.198 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:37 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:37.781+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:38.198 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:37 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:37.781+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:38.198 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:38 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:38.016+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:38.198 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:38 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:38.016+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:38.198 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:38 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:38.016+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:38.530 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:38 smithi177 podman[79196]: 2024-04-12 22:02:38.379412699 +0000 UTC m=+6.086146513 container cleanup 49cfbac67f9fb7d54c94b1595521838dc7e96c32acfb7f2a0724c51bb0de63ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5) 2024-04-12T22:02:38.530 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:38 smithi177 bash[79196]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5 2024-04-12T22:02:38.997 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:38 smithi177 podman[79213]: 2024-04-12 22:02:38.655837256 +0000 UTC m=+0.755704459 container remove 49cfbac67f9fb7d54c94b1595521838dc7e96c32acfb7f2a0724c51bb0de63ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409) 2024-04-12T22:02:38.997 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:38 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:38.790+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:38.997 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:38 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:38.790+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:38.997 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:38 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:38.790+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:39.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:39 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:39.056+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:39.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:39 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:39.056+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:39.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:39 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:39.056+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:39.280 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:38 smithi177 podman[79307]: 2024-04-12 22:02:38.899738373 +0000 UTC m=+0.017791784 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T22:02:39.280 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:39 smithi177 podman[79307]: 2024-04-12 22:02:39.016929741 +0000 UTC m=+0.134983128 container create d66196014047eef683217a088e52a0657362fe46e0040d11d6bab05372b7fe98 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-12T22:02:39.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:39 smithi177 podman[79307]: 2024-04-12 22:02:39.242706331 +0000 UTC m=+0.360759717 container init d66196014047eef683217a088e52a0657362fe46e0040d11d6bab05372b7fe98 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T22:02:39.281 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:39 smithi177 podman[79307]: 2024-04-12 22:02:39.24586541 +0000 UTC m=+0.363918798 container start d66196014047eef683217a088e52a0657362fe46e0040d11d6bab05372b7fe98 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-12T22:02:39.780 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:39 smithi177 podman[79307]: 2024-04-12 22:02:39.317823437 +0000 UTC m=+0.435876848 container attach d66196014047eef683217a088e52a0657362fe46e0040d11d6bab05372b7fe98 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-12T22:02:39.780 INFO:journalctl@ceph.osd.5.smithi177.stdout:Apr 12 22:02:39 smithi177 podman[79307]: 2024-04-12 22:02:39.533785358 +0000 UTC m=+0.651838761 container died d66196014047eef683217a088e52a0657362fe46e0040d11d6bab05372b7fe98 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T22:02:40.054 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:39 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:39.782+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:40.054 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:39 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:39.782+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:40.054 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:39 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:39.782+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:40.224 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.5.service' 2024-04-12T22:02:40.266 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:02:40.266 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-12T22:02:40.266 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-12T22:02:40.267 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.6 2024-04-12T22:02:40.329 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:40.053+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:40.330 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:40.053+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:40.330 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:40.053+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:40.591 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:40 smithi177 systemd[1]: Stopping Ceph osd.6 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:02:41.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:40.590+0000 7faf5ff13700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:02:41.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:40.590+0000 7faf5ff13700 -1 osd.6 166 *** Got signal Terminated *** 2024-04-12T22:02:41.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:40.590+0000 7faf5ff13700 -1 osd.6 166 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T22:02:41.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:40.757+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:41.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:40.757+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:41.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:40 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:40.757+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:41.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:41 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:41.042+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:41.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:41 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:41.042+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:41.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:41 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:41.042+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:42.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:41 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:41.787+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:42.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:41 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:41.787+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:42.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:41 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:41.787+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:42.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:42 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:42.058+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:42.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:42 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:42.058+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:42.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:42 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:42.058+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:43.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:42 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:42.756+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:43.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:42 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:42.756+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:43.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:42 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:42.756+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:43.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:43 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:43.036+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:43.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:43 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:43.036+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:43.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:43 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:43.036+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:43.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:43 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:43.036+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:44.280 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:43 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:43.781+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:44.280 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:43 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:43.781+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:44.280 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:43 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:43.781+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:44.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:44 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:44.023+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:44.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:44 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:44.023+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:44.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:44 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:44.023+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:44.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:44 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:44.023+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:45.280 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:44 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:44.819+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:54.077828+0000 front 2024-04-12T22:01:54.077899+0000 (oldest deadline 2024-04-12T22:02:23.478197+0000) 2024-04-12T22:02:45.280 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:44 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:44.819+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:04.579161+0000 front 2024-04-12T22:02:04.579197+0000 (oldest deadline 2024-04-12T22:02:28.079033+0000) 2024-04-12T22:02:45.280 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:44 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6[62324]: 2024-04-12T22:02:44.819+0000 7faf5c507700 -1 osd.6 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.680666+0000 front 2024-04-12T22:02:12.680696+0000 (oldest deadline 2024-04-12T22:02:35.580679+0000) 2024-04-12T22:02:45.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:45 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:45.033+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:45.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:45 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:45.033+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:45.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:45 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:45.033+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:45.281 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:45 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:45.033+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:46.383 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:46 smithi177 podman[79413]: 2024-04-12 22:02:46.101510381 +0000 UTC m=+5.613900087 container died 7ce736a7262be52343df3a8619cefbe713abfdd026ec2e5766bdd4df6fceb8bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-12T22:02:46.384 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:46 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:46.051+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:46.384 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:46 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:46.051+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:46.384 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:46 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:46.051+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:46.384 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:46 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:46.051+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:46.733 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:46 smithi177 podman[79413]: 2024-04-12 22:02:46.482342655 +0000 UTC m=+5.994732355 container cleanup 7ce736a7262be52343df3a8619cefbe713abfdd026ec2e5766bdd4df6fceb8bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=) 2024-04-12T22:02:46.733 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:46 smithi177 bash[79413]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6 2024-04-12T22:02:47.030 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:46 smithi177 podman[79434]: 2024-04-12 22:02:46.733384175 +0000 UTC m=+0.628869182 container remove 7ce736a7262be52343df3a8619cefbe713abfdd026ec2e5766bdd4df6fceb8bd (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:47 smithi177 podman[79532]: 2024-04-12 22:02:46.936333419 +0000 UTC m=+0.018874029 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:47 smithi177 podman[79532]: 2024-04-12 22:02:47.034256186 +0000 UTC m=+0.116796823 container create a7842ab77a594b53487f3cbc2c98ca748dfd3e9ad701685b057638a3bb66f2f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:47 smithi177 podman[79532]: 2024-04-12 22:02:47.278556765 +0000 UTC m=+0.361097360 container init a7842ab77a594b53487f3cbc2c98ca748dfd3e9ad701685b057638a3bb66f2f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:47 smithi177 podman[79532]: 2024-04-12 22:02:47.281744235 +0000 UTC m=+0.364284830 container start a7842ab77a594b53487f3cbc2c98ca748dfd3e9ad701685b057638a3bb66f2f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:47 smithi177 podman[79532]: 2024-04-12 22:02:47.37848275 +0000 UTC m=+0.461023372 container attach a7842ab77a594b53487f3cbc2c98ca748dfd3e9ad701685b057638a3bb66f2f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=) 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:47 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:47.052+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:47 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:47.052+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:47 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:47.052+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:47.379 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:47 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:47.052+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:47.779 INFO:journalctl@ceph.osd.6.smithi177.stdout:Apr 12 22:02:47 smithi177 podman[79532]: 2024-04-12 22:02:47.579759549 +0000 UTC m=+0.662300156 container died a7842ab77a594b53487f3cbc2c98ca748dfd3e9ad701685b057638a3bb66f2f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, ceph=True) 2024-04-12T22:02:48.207 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:48 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:48.021+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:48.207 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:48 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:48.021+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:48.207 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:48 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:48.021+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:48.207 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:48 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:48.021+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:48.230 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.6.service' 2024-04-12T22:02:48.312 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:02:48.312 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-12T22:02:48.312 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-12T22:02:48.312 DEBUG:teuthology.orchestra.run.smithi177:> sudo systemctl stop ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.7 2024-04-12T22:02:48.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:48 smithi177 systemd[1]: Stopping Ceph osd.7 for c8b0510a-f914-11ee-bc8d-c7b262605968... 2024-04-12T22:02:49.009 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:48 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:48.643+0000 7f89808d5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:02:49.009 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:48 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:48.643+0000 7f89808d5700 -1 osd.7 166 *** Got signal Terminated *** 2024-04-12T22:02:49.009 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:48 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:48.643+0000 7f89808d5700 -1 osd.7 166 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T22:02:49.009 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:49 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:49.008+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:49.009 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:49 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:49.008+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:49.009 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:49 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:49.008+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:49.009 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:49 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:49.008+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:50.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:50.016+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:50.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:50.016+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:50.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:50.016+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:50.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:50 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:50.016+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:51.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:51 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:51.014+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:51.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:51 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:51.014+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:51.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:51 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:51.014+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:51.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:51 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:51.014+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:52.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:52 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:52.019+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:52.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:52 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:52.019+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:52.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:52 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:52.019+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:52.280 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:52 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:52.019+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:53.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:53 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:53.043+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-04-12T22:01:56.482657+0000 front 2024-04-12T22:01:56.482627+0000 (oldest deadline 2024-04-12T22:02:21.782483+0000) 2024-04-12T22:02:53.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:53 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:53.043+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-04-12T22:02:01.783091+0000 front 2024-04-12T22:02:01.783069+0000 (oldest deadline 2024-04-12T22:02:26.482939+0000) 2024-04-12T22:02:53.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:53 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:53.043+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-04-12T22:02:12.184584+0000 front 2024-04-12T22:02:12.184672+0000 (oldest deadline 2024-04-12T22:02:36.284504+0000) 2024-04-12T22:02:53.530 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:53 smithi177 ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7[64593]: 2024-04-12T22:02:53.043+0000 7f897cec9700 -1 osd.7 166 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-04-12T22:02:18.585531+0000 front 2024-04-12T22:02:18.585518+0000 (oldest deadline 2024-04-12T22:02:42.685382+0000) 2024-04-12T22:02:54.410 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:54 smithi177 podman[79664]: 2024-04-12 22:02:54.147446023 +0000 UTC m=+5.614694857 container died 5bea7c19fe20273be90100e462bdf06c404320e94ec690172890f47ee83e701c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T22:02:54.777 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:54 smithi177 podman[79664]: 2024-04-12 22:02:54.509723599 +0000 UTC m=+5.976972439 container cleanup 5bea7c19fe20273be90100e462bdf06c404320e94ec690172890f47ee83e701c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T22:02:54.778 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:54 smithi177 bash[79664]: ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7 2024-04-12T22:02:55.030 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:54 smithi177 podman[79696]: 2024-04-12 22:02:54.777185163 +0000 UTC m=+0.623500200 container remove 5bea7c19fe20273be90100e462bdf06c404320e94ec690172890f47ee83e701c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T22:02:55.405 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:55 smithi177 podman[79787]: 2024-04-12 22:02:54.987435664 +0000 UTC m=+0.017588210 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T22:02:55.406 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:55 smithi177 podman[79787]: 2024-04-12 22:02:55.128118226 +0000 UTC m=+0.158270814 container create 4ae1c868877f6e97e4923bea8460ae3d690d1185c38bca68a18abd341c74388f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-12T22:02:55.406 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:55 smithi177 podman[79787]: 2024-04-12 22:02:55.330639406 +0000 UTC m=+0.360791952 container init 4ae1c868877f6e97e4923bea8460ae3d690d1185c38bca68a18abd341c74388f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux ) 2024-04-12T22:02:55.406 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:55 smithi177 podman[79787]: 2024-04-12 22:02:55.333722135 +0000 UTC m=+0.363874690 container start 4ae1c868877f6e97e4923bea8460ae3d690d1185c38bca68a18abd341c74388f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T22:02:55.406 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:55 smithi177 podman[79787]: 2024-04-12 22:02:55.405206307 +0000 UTC m=+0.435358870 container attach 4ae1c868877f6e97e4923bea8460ae3d690d1185c38bca68a18abd341c74388f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-b277feb, ceph=True) 2024-04-12T22:02:55.780 INFO:journalctl@ceph.osd.7.smithi177.stdout:Apr 12 22:02:55 smithi177 podman[79787]: 2024-04-12 22:02:55.620667102 +0000 UTC m=+0.650819651 container died 4ae1c868877f6e97e4923bea8460ae3d690d1185c38bca68a18abd341c74388f (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-c8b0510a-f914-11ee-bc8d-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, ceph=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-12T22:02:56.263 DEBUG:teuthology.orchestra.run.smithi177:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c8b0510a-f914-11ee-bc8d-c7b262605968@osd.7.service' 2024-04-12T22:02:56.306 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T22:02:56.306 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-12T22:02:56.307 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 --force --keep-logs 2024-04-12T22:05:53.487 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 --force --keep-logs 2024-04-12T22:06:12.330 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T22:06:12.366 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T22:06:12.396 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-12T22:06:12.398 DEBUG:teuthology.misc:Transferring archived files from smithi123:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541/remote/smithi123/crash 2024-04-12T22:06:12.399 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/crash -- . 2024-04-12T22:06:12.533 INFO:teuthology.orchestra.run.smithi123.stderr:tar: /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/crash: Cannot open: No such file or directory 2024-04-12T22:06:12.533 INFO:teuthology.orchestra.run.smithi123.stderr:tar: Error is not recoverable: exiting now 2024-04-12T22:06:12.535 DEBUG:teuthology.misc:Transferring archived files from smithi177:/var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541/remote/smithi177/crash 2024-04-12T22:06:12.536 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/crash -- . 2024-04-12T22:06:12.569 INFO:teuthology.orchestra.run.smithi177.stderr:tar: /var/lib/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/crash: Cannot open: No such file or directory 2024-04-12T22:06:12.569 INFO:teuthology.orchestra.run.smithi177.stderr:tar: Error is not recoverable: exiting now 2024-04-12T22:06:12.570 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-12T22:06:12.571 DEBUG:teuthology.orchestra.run.smithi123:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T22:06:12.603 INFO:teuthology.orchestra.run.smithi123.stdout:2024-04-12T21:49:24.974022+0000 mon.smithi123 (mon.0) 863 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T22:06:12.603 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-12T22:06:12.603 DEBUG:teuthology.orchestra.run.smithi123:> sudo egrep '\[SEC\]' /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T22:06:12.674 DEBUG:teuthology.orchestra.run.smithi123:> sudo egrep '\[ERR\]' /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T22:06:12.745 DEBUG:teuthology.orchestra.run.smithi123:> sudo egrep '\[WRN\]' /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T22:06:12.815 INFO:teuthology.orchestra.run.smithi123.stdout:2024-04-12T21:49:24.974022+0000 mon.smithi123 (mon.0) 863 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T22:06:12.816 INFO:tasks.cephadm:Compressing logs... 2024-04-12T22:06:12.816 DEBUG:teuthology.orchestra.run.smithi123:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-12T22:06:12.858 DEBUG:teuthology.orchestra.run.smithi177:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-12T22:06:12.884 INFO:teuthology.orchestra.run.smithi123.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-12T22:06:12.884 INFO:teuthology.orchestra.run.smithi123.stderr:: No such file or directory 2024-04-12T22:06:12.884 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.audit.log 2024-04-12T22:06:12.884 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mgr.smithi123.psjrfs.log 2024-04-12T22:06:12.884 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mon.smithi123.log 2024-04-12T22:06:12.890 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.audit.log: /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mgr.smithi123.psjrfs.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.1.log 2024-04-12T22:06:12.890 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.3.log 2024-04-12T22:06:12.890 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mon.smithi123.log: /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.0.log 2024-04-12T22:06:12.890 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.cephadm.log 2024-04-12T22:06:12.890 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log 2024-04-12T22:06:12.890 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.cephadm.log.gz 2024-04-12T22:06:12.891 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.2.log 2024-04-12T22:06:12.891 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log: 91.6% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.audit.log.gz 2024-04-12T22:06:12.891 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-volume.log 2024-04-12T22:06:12.893 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.2.log: 87.1% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log.gz 2024-04-12T22:06:12.893 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi123.poxexz.log 2024-04-12T22:06:12.894 INFO:teuthology.orchestra.run.smithi177.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-12T22:06:12.894 INFO:teuthology.orchestra.run.smithi177.stderr:: No such file or directory 2024-04-12T22:06:12.894 INFO:teuthology.orchestra.run.smithi177.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.6.log 2024-04-12T22:06:12.894 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mgr.smithi177.whybza.log 2024-04-12T22:06:12.894 INFO:teuthology.orchestra.run.smithi177.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi177.hqctzz.log 2024-04-12T22:06:12.894 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.6.log: /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mgr.smithi177.whybza.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.audit.log 2024-04-12T22:06:12.894 INFO:teuthology.orchestra.run.smithi177.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.4.log 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi177.hqctzz.log: /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.7.log 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr: 87.7% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi177.hqctzz.log.gz 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.5.log 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.cephadm.log 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.cephadm.log.gz 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mon.smithi177.log 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log: 91.7% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.audit.log.gz 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:gzip -5 --verbose -- /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-volume.log 2024-04-12T22:06:12.895 INFO:teuthology.orchestra.run.smithi177.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mon.smithi177.log: /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-volume.log: 87.1% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph.log.gz 2024-04-12T22:06:12.906 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-volume.log: /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi123.poxexz.log: 89.3% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi123.poxexz.log.gz 2024-04-12T22:06:12.908 INFO:teuthology.orchestra.run.smithi177.stderr: 91.5% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-volume.log.gz 2024-04-12T22:06:12.912 INFO:teuthology.orchestra.run.smithi123.stderr: 91.4% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-volume.log.gz 2024-04-12T22:06:12.920 INFO:teuthology.orchestra.run.smithi123.stderr: 91.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-12T22:06:12.954 INFO:teuthology.orchestra.run.smithi177.stderr: 89.7% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mgr.smithi177.whybza.log.gz 2024-04-12T22:06:13.283 INFO:teuthology.orchestra.run.smithi123.stderr: 88.4% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mgr.smithi123.psjrfs.log.gz 2024-04-12T22:06:13.382 INFO:teuthology.orchestra.run.smithi177.stderr: 92.6% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mon.smithi177.log.gz 2024-04-12T22:06:14.244 INFO:teuthology.orchestra.run.smithi123.stderr: 92.0% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-mon.smithi123.log.gz 2024-04-12T22:06:25.419 INFO:teuthology.orchestra.run.smithi123.stderr: 94.3% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.2.log.gz 2024-04-12T22:06:29.177 INFO:teuthology.orchestra.run.smithi177.stderr: 93.9% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.4.log.gz 2024-04-12T22:06:30.155 INFO:teuthology.orchestra.run.smithi123.stderr: 94.1% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.0.log.gz 2024-04-12T22:06:30.290 INFO:teuthology.orchestra.run.smithi177.stderr: 94.2% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.6.log.gz 2024-04-12T22:06:33.099 INFO:teuthology.orchestra.run.smithi123.stderr: 94.0% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.1.log.gz 2024-04-12T22:06:33.324 INFO:teuthology.orchestra.run.smithi177.stderr: 94.2% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.5.log.gz 2024-04-12T22:06:34.058 INFO:teuthology.orchestra.run.smithi123.stderr: 94.1% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.3.log.gz 2024-04-12T22:06:34.059 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-04-12T22:06:34.060 INFO:teuthology.orchestra.run.smithi123.stderr:real 0m21.189s 2024-04-12T22:06:34.060 INFO:teuthology.orchestra.run.smithi123.stderr:user 1m10.189s 2024-04-12T22:06:34.060 INFO:teuthology.orchestra.run.smithi123.stderr:sys 0m2.711s 2024-04-12T22:06:38.817 INFO:teuthology.orchestra.run.smithi177.stderr: 94.1% -- replaced with /var/log/ceph/c8b0510a-f914-11ee-bc8d-c7b262605968/ceph-osd.7.log.gz 2024-04-12T22:06:38.819 INFO:teuthology.orchestra.run.smithi177.stderr: 2024-04-12T22:06:38.819 INFO:teuthology.orchestra.run.smithi177.stderr:real 0m25.947s 2024-04-12T22:06:38.819 INFO:teuthology.orchestra.run.smithi177.stderr:user 1m17.500s 2024-04-12T22:06:38.819 INFO:teuthology.orchestra.run.smithi177.stderr:sys 0m3.026s 2024-04-12T22:06:38.819 INFO:tasks.cephadm:Archiving logs... 2024-04-12T22:06:38.820 DEBUG:teuthology.misc:Transferring archived files from smithi123:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541/remote/smithi123/log 2024-04-12T22:06:38.821 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-12T22:06:48.838 DEBUG:teuthology.misc:Transferring archived files from smithi177:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541/remote/smithi177/log 2024-04-12T22:06:48.839 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-12T22:06:59.932 INFO:tasks.cephadm:Removing cluster... 2024-04-12T22:06:59.933 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 --force 2024-04-12T22:07:00.317 DEBUG:teuthology.orchestra.run.smithi177:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c8b0510a-f914-11ee-bc8d-c7b262605968 --force 2024-04-12T22:07:00.705 INFO:tasks.cephadm:Removing cephadm ... 2024-04-12T22:07:00.705 DEBUG:teuthology.orchestra.run.smithi123:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-12T22:07:00.721 DEBUG:teuthology.orchestra.run.smithi177:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-12T22:07:00.736 INFO:tasks.cephadm:Teardown complete 2024-04-12T22:07:00.737 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-12T22:07:00.747 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-12T22:07:00.748 DEBUG:teuthology.orchestra.run.smithi123:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T22:07:00.764 DEBUG:teuthology.orchestra.run.smithi177:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T22:07:00.775 INFO:teuthology.orchestra.run.smithi123.stderr:bash: line 1: ntpq: command not found 2024-04-12T22:07:00.779 INFO:teuthology.orchestra.run.smithi123.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T22:07:00.780 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================== 2024-04-12T22:07:00.780 INFO:teuthology.orchestra.run.smithi123.stdout:^- hv01.front.sepia.ceph.com 2 6 377 54 +1125us[+1143us] +/- 361ms 2024-04-12T22:07:00.780 INFO:teuthology.orchestra.run.smithi123.stdout:^* hv02.front.sepia.ceph.com 3 6 377 52 -430us[ -412us] +/- 55ms 2024-04-12T22:07:00.780 INFO:teuthology.orchestra.run.smithi123.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 +578us[ +578us] +/- 65ms 2024-04-12T22:07:00.780 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:07:00.790 INFO:teuthology.orchestra.run.smithi177.stderr:bash: line 1: ntpq: command not found 2024-04-12T22:07:00.794 INFO:teuthology.orchestra.run.smithi177.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T22:07:00.795 INFO:teuthology.orchestra.run.smithi177.stdout:=============================================================================== 2024-04-12T22:07:00.795 INFO:teuthology.orchestra.run.smithi177.stdout:^- hv01.front.sepia.ceph.com 2 6 377 50 +1123us[+1123us] +/- 361ms 2024-04-12T22:07:00.795 INFO:teuthology.orchestra.run.smithi177.stdout:^* hv02.front.sepia.ceph.com 3 6 377 54 -418us[ -376us] +/- 55ms 2024-04-12T22:07:00.795 INFO:teuthology.orchestra.run.smithi177.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 53 +570us[ +570us] +/- 65ms 2024-04-12T22:07:00.795 INFO:teuthology.orchestra.run.smithi177.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:07:00.795 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-12T22:07:00.806 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-12T22:07:00.807 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-12T22:07:00.817 DEBUG:teuthology.orchestra.run.smithi123:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-12T22:07:00.905 DEBUG:teuthology.orchestra.run.smithi177:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-12T22:07:00.984 DEBUG:teuthology.orchestra.run.smithi123:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-12T22:07:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T22:07:01.015 DEBUG:teuthology.orchestra.run.smithi177:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-12T22:07:01.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T22:07:01.046 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-12T22:07:01.056 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-12T22:07:01.065 INFO:teuthology.task.internal:Duration was 2068.687277 seconds 2024-04-12T22:07:01.065 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-12T22:07:01.076 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-12T22:07:01.077 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-12T22:07:01.079 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-12T22:07:01.119 INFO:teuthology.orchestra.run.smithi123.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T22:07:01.129 INFO:teuthology.orchestra.run.smithi177.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T22:07:01.571 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-12T22:07:01.571 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi123.front.sepia.ceph.com 2024-04-12T22:07:01.571 DEBUG:teuthology.orchestra.run.smithi123:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-12T22:07:01.591 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi177.front.sepia.ceph.com 2024-04-12T22:07:01.592 DEBUG:teuthology.orchestra.run.smithi177:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-12T22:07:01.638 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-12T22:07:01.638 DEBUG:teuthology.orchestra.run.smithi123:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-12T22:07:01.640 DEBUG:teuthology.orchestra.run.smithi177:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-12T22:07:01.707 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-12T22:07:01.707 DEBUG:teuthology.orchestra.run.smithi123:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-12T22:07:01.709 DEBUG:teuthology.orchestra.run.smithi177:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-12T22:07:15.456 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-12T22:07:15.467 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-12T22:07:15.467 DEBUG:teuthology.orchestra.run.smithi123:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-12T22:07:15.494 DEBUG:teuthology.orchestra.run.smithi177:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-12T22:07:15.526 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-12T22:07:15.537 DEBUG:teuthology.orchestra.run.smithi123:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-12T22:07:15.541 DEBUG:teuthology.orchestra.run.smithi177:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-12T22:07:15.562 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern = core 2024-04-12T22:07:15.596 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern = core 2024-04-12T22:07:15.659 DEBUG:teuthology.orchestra.run.smithi123:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-12T22:07:15.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T22:07:15.714 DEBUG:teuthology.orchestra.run.smithi177:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-12T22:07:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T22:07:15.730 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-12T22:07:15.741 INFO:teuthology.task.internal:Transferring archived files... 2024-04-12T22:07:15.745 DEBUG:teuthology.misc:Transferring archived files from smithi123:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541/remote/smithi123 2024-04-12T22:07:15.745 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-12T22:07:15.819 DEBUG:teuthology.misc:Transferring archived files from smithi177:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541/remote/smithi177 2024-04-12T22:07:15.819 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-12T22:07:15.865 INFO:teuthology.task.internal:Removing archive directory... 2024-04-12T22:07:15.866 DEBUG:teuthology.orchestra.run.smithi123:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-12T22:07:15.870 DEBUG:teuthology.orchestra.run.smithi177:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-12T22:07:15.915 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-12T22:07:15.928 INFO:teuthology.task.internal:Not uploading archives. 2024-04-12T22:07:15.928 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-12T22:07:15.939 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-12T22:07:15.939 DEBUG:teuthology.orchestra.run.smithi123:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-12T22:07:15.942 DEBUG:teuthology.orchestra.run.smithi177:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-12T22:07:15.954 INFO:teuthology.orchestra.run.smithi123.stdout: 266041 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 12 22:07 /home/ubuntu/cephtest 2024-04-12T22:07:15.971 INFO:teuthology.orchestra.run.smithi177.stdout: 265595 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 12 22:07 /home/ubuntu/cephtest 2024-04-12T22:07:15.972 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-12T22:07:15.983 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-12T22:07:16.006 INFO:teuthology.nuke:Checking targets against current locks 2024-04-12T22:07:16.033 DEBUG:teuthology.nuke:shortname: smithi123 2024-04-12T22:07:16.033 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T22:07:16.058 DEBUG:teuthology.nuke:shortname: smithi177 2024-04-12T22:07:16.058 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T22:07:16.061 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi123.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 21:24:37.645509', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGmn6Phy9Qyz8bcmSfXEWGoGASyjs+Z70dhNTVJ+VNRDnDca15Dme2F1oo44XBwBm3Bws9Latr+JelBnYJF8Mfk='} 2024-04-12T22:07:16.087 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi177.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654541', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 21:24:37.646798', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP5navxQ0Zgey4yHcCk2usMWx4nnxrcWaXPPRSetGO/ESja/FSz912f94o5bW8g5Nze5LHJnfSHoODgUvdLqR/w='} 2024-04-12T22:07:16.117 INFO:teuthology.orchestra.console.smithi123:Power off 2024-04-12T22:07:16.117 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-12T22:07:16.141 DEBUG:teuthology.orchestra.console.smithi123:power off output: Chassis Power Control: Down/Off 2024-04-12T22:07:16.141 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T22:07:16.158 INFO:teuthology.orchestra.console.smithi177:Power off 2024-04-12T22:07:16.159 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-12T22:07:16.170 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is on 2024-04-12T22:07:16.184 DEBUG:teuthology.orchestra.console.smithi177:power off output: Chassis Power Control: Down/Off 2024-04-12T22:07:16.184 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T22:07:16.206 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is on 2024-04-12T22:07:20.171 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T22:07:20.207 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T22:07:20.284 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is on 2024-04-12T22:07:20.316 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is on 2024-04-12T22:07:24.287 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T22:07:24.317 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T22:07:24.396 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is on 2024-04-12T22:07:24.426 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is off 2024-04-12T22:07:24.527 INFO:teuthology.orchestra.console.smithi177:Power off completed 2024-04-12T22:07:28.398 DEBUG:teuthology.orchestra.console.smithi123:pexpect command: ipmitool -H smithi123.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T22:07:28.508 DEBUG:teuthology.orchestra.console.smithi123:check power output: Chassis Power is off 2024-04-12T22:07:28.609 INFO:teuthology.orchestra.console.smithi123:Power off completed 2024-04-12T22:07:28.713 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2068.6872766017914 failure_reason: '"2024-04-12T21:49:24.974022+0000 mon.smithi123 (mon.0) 863 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-04-12T22:07:28.713 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-12T22:07:28.787 INFO:teuthology.run:FAIL