2024-09-13T21:24:46.533 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T21:24:46.551 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T21:24:46.647 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7904199' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 3819 sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 sleep_before_teardown: 0 subset: 43/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 targets: smithi154.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFNMv6MO/qqf6ViXMjJRw+KABNlJLl3pv4NO1nwdwajtwFmD878NaWaK4J0wL+ZE+8CYcJ8Fa/OF9DkarAgi4gQ= smithi181.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKNWUHG1ZxMNadUSngQk4U3vxUJ/t4bv1s2RzUC3ymA7z8Cz0F3QGXokHG/JSBu2BycoEm3XQYq7nAX0JneleQg= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_21:08:03 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-13T21:24:46.648 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa; will attempt to use it 2024-09-13T21:24:46.648 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa/tasks 2024-09-13T21:24:46.649 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T21:24:46.650 INFO:teuthology.task.internal:Checking packages... 2024-09-13T21:24:46.671 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'ecab7be58963a5bd03910cd73eef66405d7fdd50' 2024-09-13T21:24:46.672 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T21:24:46.672 INFO:teuthology.packaging:ref: None 2024-09-13T21:24:46.672 INFO:teuthology.packaging:tag: None 2024-09-13T21:24:46.672 INFO:teuthology.packaging:branch: squid 2024-09-13T21:24:46.672 INFO:teuthology.packaging:sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T21:24:46.672 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-13T21:24:46.912 INFO:teuthology.task.internal:Found packages for ceph version 19.1.1-324.gecab7be5 2024-09-13T21:24:46.913 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T21:24:46.919 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T21:24:46.919 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T21:24:47.008 INFO:teuthology.task.internal:Saving configuration 2024-09-13T21:24:47.025 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T21:24:47.031 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T21:24:47.061 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi154.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 21:19:32.371255', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFNMv6MO/qqf6ViXMjJRw+KABNlJLl3pv4NO1nwdwajtwFmD878NaWaK4J0wL+ZE+8CYcJ8Fa/OF9DkarAgi4gQ='} 2024-09-13T21:24:47.090 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 21:19:32.372409', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKNWUHG1ZxMNadUSngQk4U3vxUJ/t4bv1s2RzUC3ymA7z8Cz0F3QGXokHG/JSBu2BycoEm3XQYq7nAX0JneleQg='} 2024-09-13T21:24:47.090 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T21:24:47.096 INFO:teuthology.task.internal:roles: ubuntu@smithi154.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-13T21:24:47.096 INFO:teuthology.task.internal:roles: ubuntu@smithi181.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-13T21:24:47.096 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T21:24:47.171 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f1562453ac0>, signals=[15]) 2024-09-13T21:24:47.171 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T21:24:47.185 INFO:teuthology.task.internal:Opening connections... 2024-09-13T21:24:47.185 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:24:47.186 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:24:47.273 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi181.front.sepia.ceph.com 2024-09-13T21:24:47.274 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:24:47.362 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T21:24:47.368 DEBUG:teuthology.orchestra.run.smithi154:> uname -m 2024-09-13T21:24:47.387 INFO:teuthology.orchestra.run.smithi154.stdout:x86_64 2024-09-13T21:24:47.387 DEBUG:teuthology.orchestra.run.smithi154:> cat /etc/os-release 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:NAME="CentOS Stream" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:VERSION="9" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:ID="centos" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:ID_LIKE="rhel fedora" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:VERSION_ID="9" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:PLATFORM_ID="platform:el9" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:ANSI_COLOR="0;31" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:LOGO="fedora-logo-icon" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:HOME_URL="https://centos.org/" 2024-09-13T21:24:47.446 INFO:teuthology.orchestra.run.smithi154.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T21:24:47.447 INFO:teuthology.orchestra.run.smithi154.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T21:24:47.447 INFO:teuthology.orchestra.run.smithi154.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T21:24:47.447 INFO:teuthology.lock.ops:Updating smithi154.front.sepia.ceph.com on lock server 2024-09-13T21:24:47.480 DEBUG:teuthology.orchestra.run.smithi181:> uname -m 2024-09-13T21:24:47.499 INFO:teuthology.orchestra.run.smithi181.stdout:x86_64 2024-09-13T21:24:47.499 DEBUG:teuthology.orchestra.run.smithi181:> cat /etc/os-release 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:NAME="CentOS Stream" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION="9" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:ID="centos" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:ID_LIKE="rhel fedora" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION_ID="9" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:PLATFORM_ID="platform:el9" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:ANSI_COLOR="0;31" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:LOGO="fedora-logo-icon" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T21:24:47.557 INFO:teuthology.orchestra.run.smithi181.stdout:HOME_URL="https://centos.org/" 2024-09-13T21:24:47.558 INFO:teuthology.orchestra.run.smithi181.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T21:24:47.558 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T21:24:47.558 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T21:24:47.558 INFO:teuthology.lock.ops:Updating smithi181.front.sepia.ceph.com on lock server 2024-09-13T21:24:47.580 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T21:24:47.588 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T21:24:47.592 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T21:24:47.593 DEBUG:teuthology.orchestra.run.smithi154:> test '!' -e /home/ubuntu/cephtest 2024-09-13T21:24:47.595 DEBUG:teuthology.orchestra.run.smithi181:> test '!' -e /home/ubuntu/cephtest 2024-09-13T21:24:47.612 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T21:24:47.618 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T21:24:47.618 DEBUG:teuthology.orchestra.run.smithi154:> test -z $(ls -A /var/lib/ceph) 2024-09-13T21:24:47.653 DEBUG:teuthology.orchestra.run.smithi181:> test -z $(ls -A /var/lib/ceph) 2024-09-13T21:24:47.780 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T21:24:47.841 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T21:24:47.855 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T21:24:47.856 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T21:24:47.856 DEBUG:teuthology.orchestra.run.smithi154:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T21:24:47.856 DEBUG:teuthology.orchestra.run.smithi181:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T21:24:47.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T21:24:47.874 DEBUG:teuthology.orchestra.run.smithi154:> uname -r 2024-09-13T21:24:47.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T21:24:47.875 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2024-09-13T21:24:47.932 INFO:teuthology.orchestra.run.smithi154.stdout:5.14.0-503.el9.x86_64 2024-09-13T21:24:47.932 INFO:teuthology.task.kernel:Running kernel on smithi154: 5.14.0-503.el9.x86_64 2024-09-13T21:24:47.932 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum install -y kernel 2024-09-13T21:24:47.933 INFO:teuthology.orchestra.run.smithi181.stdout:5.14.0-503.el9.x86_64 2024-09-13T21:24:47.934 INFO:teuthology.task.kernel:Running kernel on smithi181: 5.14.0-503.el9.x86_64 2024-09-13T21:24:47.934 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum install -y kernel 2024-09-13T21:24:50.321 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.2 MB 00:00 2024-09-13T21:24:50.448 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.2 MB 00:00 2024-09-13T21:24:52.089 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - AppStream 31 MB/s | 20 MB 00:00 2024-09-13T21:24:52.349 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - AppStream 27 MB/s | 20 MB 00:00 2024-09-13T21:24:56.658 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.5 MB 00:00 2024-09-13T21:24:57.272 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - CRB 4.7 MB/s | 6.5 MB 00:01 2024-09-13T21:24:58.342 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - Extras packages 38 kB/s | 19 kB 00:00 2024-09-13T21:24:58.673 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - Extras packages 101 kB/s | 19 kB 00:00 2024-09-13T21:24:59.097 INFO:teuthology.orchestra.run.smithi154.stdout:Extra Packages for Enterprise Linux 36 MB/s | 23 MB 00:00 2024-09-13T21:24:59.802 INFO:teuthology.orchestra.run.smithi181.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-13T21:25:04.577 INFO:teuthology.orchestra.run.smithi154.stdout:lab-extras 26 kB/s | 1.7 kB 00:00 2024-09-13T21:25:05.191 INFO:teuthology.orchestra.run.smithi181.stdout:lab-extras 35 kB/s | 1.7 kB 00:00 2024-09-13T21:25:06.609 INFO:teuthology.orchestra.run.smithi154.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T21:25:06.609 INFO:teuthology.orchestra.run.smithi154.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T21:25:06.705 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-09-13T21:25:06.717 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-09-13T21:25:06.717 INFO:teuthology.orchestra.run.smithi154.stdout: Package Architecture Version Repository Size 2024-09-13T21:25:06.717 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-09-13T21:25:06.717 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout:Install 4 Packages 2024-09-13T21:25:06.718 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:25:06.719 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 79 M 2024-09-13T21:25:06.719 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 126 M 2024-09-13T21:25:06.719 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-09-13T21:25:06.937 INFO:teuthology.orchestra.run.smithi154.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 228 kB/s | 23 kB 00:00 2024-09-13T21:25:07.173 INFO:teuthology.orchestra.run.smithi181.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T21:25:07.173 INFO:teuthology.orchestra.run.smithi181.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T21:25:07.260 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout: Package Architecture Version Repository Size 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:25:07.268 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-09-13T21:25:07.269 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-09-13T21:25:07.269 INFO:teuthology.orchestra.run.smithi181.stdout:Install 4 Packages 2024-09-13T21:25:07.269 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:25:07.269 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 79 M 2024-09-13T21:25:07.269 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 126 M 2024-09-13T21:25:07.269 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-09-13T21:25:07.354 INFO:teuthology.orchestra.run.smithi154.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 30 MB/s | 16 MB 00:00 2024-09-13T21:25:07.609 INFO:teuthology.orchestra.run.smithi181.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 77 kB/s | 23 kB 00:00 2024-09-13T21:25:07.804 INFO:teuthology.orchestra.run.smithi154.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 36 MB/s | 35 MB 00:00 2024-09-13T21:25:08.130 INFO:teuthology.orchestra.run.smithi154.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-13T21:25:08.130 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-09-13T21:25:08.130 INFO:teuthology.orchestra.run.smithi154.stdout:Total 56 MB/s | 79 MB 00:01 2024-09-13T21:25:08.813 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-09-13T21:25:09.160 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-09-13T21:25:09.160 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-09-13T21:25:10.134 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-09-13T21:25:10.135 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-09-13T21:25:12.575 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-09-13T21:25:13.034 INFO:teuthology.orchestra.run.smithi181.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 2.7 MB/s | 16 MB 00:05 2024-09-13T21:25:13.283 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T21:25:13.364 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:25:16.154 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:25:16.291 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:25:19.628 INFO:teuthology.orchestra.run.smithi181.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 2.4 MB/s | 29 MB 00:12 2024-09-13T21:25:20.316 INFO:teuthology.orchestra.run.smithi181.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 2.7 MB/s | 35 MB 00:13 2024-09-13T21:25:20.317 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-09-13T21:25:20.317 INFO:teuthology.orchestra.run.smithi181.stdout:Total 6.0 MB/s | 79 MB 00:13 2024-09-13T21:25:20.999 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-09-13T21:25:21.115 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:25:21.236 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:25:21.358 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-09-13T21:25:21.457 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-09-13T21:25:22.320 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-09-13T21:25:22.320 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-09-13T21:25:24.733 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-09-13T21:25:25.375 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T21:25:25.458 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:25:26.093 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:25:28.232 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:25:28.362 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:25:33.197 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:25:33.321 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:25:38.162 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:04.570 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:04.616 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:05.051 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:05.051 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T21:26:05.052 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:26:05.052 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:26:05.927 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-09-13T21:26:06.263 DEBUG:teuthology.orchestra.run.smithi154:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T21:26:06.392 INFO:teuthology.orchestra.run.smithi154.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T21:26:06.392 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T21:26:06.392 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T21:26:06.392 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T21:26:06.392 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T21:26:06.393 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-13T21:26:06.393 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi154.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T21:26:06.393 DEBUG:teuthology.orchestra.run.smithi154:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T21:26:06.451 INFO:teuthology.orchestra.run.smithi154.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T21:26:06.452 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T21:26:06.453 DEBUG:teuthology.orchestra.run.smithi154:> sudo rpm -qi grub2-tools 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Name : grub2-tools 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Epoch : 1 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Version : 2.06 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Release : 82.el9 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Architecture: x86_64 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Group : Unspecified 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Size : 8274670 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:License : GPLv3+ 2024-09-13T21:26:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:Packager : builder@centos.org 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:Vendor : CentOS 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:Summary : Support tools for GRUB. 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:Description : 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T21:26:06.487 INFO:teuthology.orchestra.run.smithi154.stdout:hardware devices. 2024-09-13T21:26:06.488 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:26:06.488 INFO:teuthology.orchestra.run.smithi154.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T21:26:06.488 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T21:26:06.489 INFO:teuthology.task.kernel:Updating grub on smithi154 to boot 5.14.0-505.el9.x86_64 2024-09-13T21:26:06.489 DEBUG:teuthology.orchestra.run.smithi154:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T21:26:07.145 INFO:teuthology.orchestra.run.smithi154.stderr:Generating grub configuration file ... 2024-09-13T21:26:08.462 INFO:teuthology.orchestra.run.smithi154.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T21:26:08.495 INFO:teuthology.orchestra.run.smithi154.stderr:done 2024-09-13T21:26:08.505 DEBUG:teuthology.orchestra.run.smithi154:> mktemp 2024-09-13T21:26:08.521 INFO:teuthology.orchestra.run.smithi154.stdout:/tmp/tmp.ZOWr3J4226 2024-09-13T21:26:08.522 DEBUG:teuthology.orchestra.run.smithi154:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ZOWr3J4226 2024-09-13T21:26:08.599 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod 0666 /tmp/tmp.ZOWr3J4226 2024-09-13T21:26:08.739 DEBUG:teuthology.orchestra.remote:smithi154:/tmp/tmp.ZOWr3J4226 is 6KB 2024-09-13T21:26:08.788 DEBUG:teuthology.orchestra.run.smithi154:> rm -fr /tmp/tmp.ZOWr3J4226 2024-09-13T21:26:08.803 DEBUG:teuthology.orchestra.run.smithi154:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T21:26:08.872 INFO:teuthology.orchestra.run.smithi154.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T21:26:08.872 INFO:teuthology.orchestra.run.smithi154.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T21:26:08.872 INFO:teuthology.orchestra.run.smithi154.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T21:26:08.873 INFO:teuthology.orchestra.run.smithi154.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T21:26:08.874 DEBUG:teuthology.orchestra.run.smithi154:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T21:26:09.021 DEBUG:teuthology.orchestra.run.smithi154:> sudo shutdown -r now 2024-09-13T21:26:16.427 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:16.466 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:16.781 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T21:26:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T21:26:16.782 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:26:17.637 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-09-13T21:26:17.998 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T21:26:18.084 INFO:teuthology.orchestra.run.smithi181.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T21:26:18.085 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T21:26:18.085 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T21:26:18.085 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T21:26:18.085 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T21:26:18.085 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-13T21:26:18.085 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi181.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T21:26:18.085 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T21:26:18.144 INFO:teuthology.orchestra.run.smithi181.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T21:26:18.144 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T21:26:18.145 DEBUG:teuthology.orchestra.run.smithi181:> sudo rpm -qi grub2-tools 2024-09-13T21:26:18.177 INFO:teuthology.orchestra.run.smithi181.stdout:Name : grub2-tools 2024-09-13T21:26:18.177 INFO:teuthology.orchestra.run.smithi181.stdout:Epoch : 1 2024-09-13T21:26:18.177 INFO:teuthology.orchestra.run.smithi181.stdout:Version : 2.06 2024-09-13T21:26:18.177 INFO:teuthology.orchestra.run.smithi181.stdout:Release : 82.el9 2024-09-13T21:26:18.177 INFO:teuthology.orchestra.run.smithi181.stdout:Architecture: x86_64 2024-09-13T21:26:18.177 INFO:teuthology.orchestra.run.smithi181.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T21:26:18.177 INFO:teuthology.orchestra.run.smithi181.stdout:Group : Unspecified 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Size : 8274670 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:License : GPLv3+ 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Packager : builder@centos.org 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Vendor : CentOS 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Summary : Support tools for GRUB. 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:Description : 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T21:26:18.178 INFO:teuthology.orchestra.run.smithi181.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T21:26:18.179 INFO:teuthology.orchestra.run.smithi181.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T21:26:18.179 INFO:teuthology.orchestra.run.smithi181.stdout:hardware devices. 2024-09-13T21:26:18.179 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:26:18.179 INFO:teuthology.orchestra.run.smithi181.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T21:26:18.180 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T21:26:18.180 INFO:teuthology.task.kernel:Updating grub on smithi181 to boot 5.14.0-505.el9.x86_64 2024-09-13T21:26:18.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T21:26:18.814 INFO:teuthology.orchestra.run.smithi181.stderr:Generating grub configuration file ... 2024-09-13T21:26:20.178 INFO:teuthology.orchestra.run.smithi181.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T21:26:20.231 INFO:teuthology.orchestra.run.smithi181.stderr:done 2024-09-13T21:26:20.233 DEBUG:teuthology.orchestra.run.smithi181:> mktemp 2024-09-13T21:26:20.249 INFO:teuthology.orchestra.run.smithi181.stdout:/tmp/tmp.GbzpDsxWvr 2024-09-13T21:26:20.249 DEBUG:teuthology.orchestra.run.smithi181:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.GbzpDsxWvr 2024-09-13T21:26:20.315 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0666 /tmp/tmp.GbzpDsxWvr 2024-09-13T21:26:20.453 DEBUG:teuthology.orchestra.remote:smithi181:/tmp/tmp.GbzpDsxWvr is 6KB 2024-09-13T21:26:20.502 DEBUG:teuthology.orchestra.run.smithi181:> rm -fr /tmp/tmp.GbzpDsxWvr 2024-09-13T21:26:20.518 DEBUG:teuthology.orchestra.run.smithi181:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T21:26:20.581 INFO:teuthology.orchestra.run.smithi181.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T21:26:20.581 INFO:teuthology.orchestra.run.smithi181.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T21:26:20.582 INFO:teuthology.orchestra.run.smithi181.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T21:26:20.582 INFO:teuthology.orchestra.run.smithi181.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T21:26:20.583 DEBUG:teuthology.orchestra.run.smithi181:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T21:26:20.747 DEBUG:teuthology.orchestra.run.smithi181:> sudo shutdown -r now 2024-09-13T21:26:39.037 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-13T21:26:39.038 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-09-13T21:26:39.039 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:26:50.761 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-13T21:26:50.761 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-13T21:26:50.762 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:27:13.730 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.154 2024-09-13T21:27:22.733 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-09-13T21:27:22.735 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:27:50.797 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T21:27:59.805 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-13T21:27:59.806 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:28:00.253 DEBUG:teuthology.orchestra.run.smithi181:> true 2024-09-13T21:28:00.579 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-13T21:28:00.579 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T21:28:00.579 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2024-09-13T21:28:00.593 INFO:teuthology.orchestra.run.smithi181.stdout:5.14.0-505.el9.x86_64 2024-09-13T21:28:00.594 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T21:28:00.594 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T21:28:00.594 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T21:28:01.595 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-13T21:28:01.596 DEBUG:teuthology.orchestra.run.smithi181:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T21:28:01.722 INFO:teuthology.orchestra.run.smithi181.stdout:ttyS1 2024-09-13T21:28:01.739 DEBUG:teuthology.parallel:result is None 2024-09-13T21:28:22.745 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T21:28:34.747 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-09-13T21:28:34.748 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:28:35.201 DEBUG:teuthology.orchestra.run.smithi154:> true 2024-09-13T21:28:35.584 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-09-13T21:28:35.584 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T21:28:35.584 DEBUG:teuthology.orchestra.run.smithi154:> uname -r 2024-09-13T21:28:35.601 INFO:teuthology.orchestra.run.smithi154.stdout:5.14.0-505.el9.x86_64 2024-09-13T21:28:35.601 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T21:28:35.601 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T21:28:35.602 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T21:28:36.603 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-13T21:28:36.603 DEBUG:teuthology.orchestra.run.smithi154:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T21:28:36.693 INFO:teuthology.orchestra.run.smithi154.stdout:ttyS1 2024-09-13T21:28:36.717 DEBUG:teuthology.parallel:result is None 2024-09-13T21:28:36.717 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T21:28:36.724 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T21:28:36.724 DEBUG:teuthology.orchestra.run.smithi154:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T21:28:36.726 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T21:28:36.749 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T21:28:36.756 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T21:28:36.765 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T21:28:36.765 DEBUG:teuthology.orchestra.run.smithi154:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T21:28:36.785 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T21:28:36.816 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T21:28:36.822 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T21:28:36.822 DEBUG:teuthology.orchestra.run.smithi154:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T21:28:36.855 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T21:28:36.886 INFO:teuthology.orchestra.run.smithi154.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T21:28:36.890 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T21:28:36.898 INFO:teuthology.orchestra.run.smithi154.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T21:28:36.902 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T21:28:36.903 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T21:28:36.911 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T21:28:36.911 DEBUG:teuthology.orchestra.run.smithi154:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T21:28:36.942 DEBUG:teuthology.orchestra.run.smithi181:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T21:28:36.971 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T21:28:36.980 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T21:28:36.980 DEBUG:teuthology.orchestra.run.smithi154:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T21:28:37.011 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T21:28:37.028 DEBUG:teuthology.orchestra.run.smithi154:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T21:28:37.114 DEBUG:teuthology.orchestra.run.smithi154:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T21:28:37.160 DEBUG:teuthology.orchestra.run.smithi154:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T21:28:37.250 DEBUG:teuthology.orchestra.run.smithi154:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T21:28:37.284 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:28:37.284 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T21:28:37.349 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T21:28:37.400 DEBUG:teuthology.orchestra.run.smithi181:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T21:28:37.460 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T21:28:37.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T21:28:37.548 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-13T21:28:37.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T21:28:37.615 DEBUG:teuthology.orchestra.run.smithi154:> sudo service rsyslog restart 2024-09-13T21:28:37.618 DEBUG:teuthology.orchestra.run.smithi181:> sudo service rsyslog restart 2024-09-13T21:28:37.664 INFO:teuthology.orchestra.run.smithi154.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T21:28:37.702 INFO:teuthology.orchestra.run.smithi181.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T21:28:38.119 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T21:28:38.128 INFO:teuthology.task.internal:Starting timer... 2024-09-13T21:28:38.128 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T21:28:38.139 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T21:28:38.155 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-09-13T21:28:38.155 DEBUG:teuthology.orchestra.run.smithi154:> sudo service auditd rotate 2024-09-13T21:28:38.210 INFO:teuthology.orchestra.run.smithi154.stdout:Rotating logs: 2024-09-13T21:28:38.212 DEBUG:teuthology.orchestra.run.smithi181:> sudo service auditd rotate 2024-09-13T21:28:38.293 INFO:teuthology.orchestra.run.smithi181.stdout:Rotating logs: 2024-09-13T21:28:38.295 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T21:28:38.295 DEBUG:teuthology.orchestra.run.smithi154:> /usr/sbin/getenforce 2024-09-13T21:28:38.320 INFO:teuthology.orchestra.run.smithi154.stdout:Permissive 2024-09-13T21:28:38.320 DEBUG:teuthology.orchestra.run.smithi181:> /usr/sbin/getenforce 2024-09-13T21:28:38.346 INFO:teuthology.orchestra.run.smithi181.stdout:Permissive 2024-09-13T21:28:38.347 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi154.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi181.front.sepia.ceph.com': 'permissive'} 2024-09-13T21:28:38.347 DEBUG:teuthology.orchestra.run.smithi154:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T21:28:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T21:28:38.386 DEBUG:teuthology.orchestra.run.smithi181:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T21:28:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T21:28:38.413 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T21:28:38.413 DEBUG:teuthology.orchestra.run.smithi154:> sudo /usr/sbin/setenforce permissive 2024-09-13T21:28:38.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo /usr/sbin/setenforce permissive 2024-09-13T21:28:38.487 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T21:28:38.497 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T21:28:38.509 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T21:28:38.513 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi154.front.sepia.ceph.com,smithi181.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T21:32:28.524 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi154.front.sepia.ceph.com'), Remote(name='ubuntu@smithi181.front.sepia.ceph.com')] 2024-09-13T21:32:28.525 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-09-13T21:32:28.526 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:32:28.613 DEBUG:teuthology.orchestra.run.smithi154:> true 2024-09-13T21:32:28.705 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-09-13T21:32:28.705 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-13T21:32:28.706 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T21:32:28.802 DEBUG:teuthology.orchestra.run.smithi181:> true 2024-09-13T21:32:28.882 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-09-13T21:32:28.882 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T21:32:28.895 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T21:32:28.895 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T21:32:28.895 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T21:32:28.899 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T21:32:28.899 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T21:32:28.925 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T21:32:28.940 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T21:32:28.952 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T21:32:28.966 INFO:teuthology.orchestra.run.smithi154.stderr:sudo: ntpd: command not found 2024-09-13T21:32:28.967 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T21:32:28.977 INFO:teuthology.orchestra.run.smithi154.stdout:506 Cannot talk to daemon 2024-09-13T21:32:28.991 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T21:32:28.992 INFO:teuthology.orchestra.run.smithi181.stderr:sudo: ntpd: command not found 2024-09-13T21:32:29.003 INFO:teuthology.orchestra.run.smithi181.stdout:506 Cannot talk to daemon 2024-09-13T21:32:29.005 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T21:32:29.017 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T21:32:29.031 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T21:32:29.059 INFO:teuthology.orchestra.run.smithi154.stderr:bash: line 1: ntpq: command not found 2024-09-13T21:32:29.062 INFO:teuthology.orchestra.run.smithi154.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T21:32:29.062 INFO:teuthology.orchestra.run.smithi154.stdout:=============================================================================== 2024-09-13T21:32:29.062 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:32:29.062 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:32:29.062 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:32:29.063 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:32:29.095 INFO:teuthology.orchestra.run.smithi181.stderr:bash: line 1: ntpq: command not found 2024-09-13T21:32:29.099 INFO:teuthology.orchestra.run.smithi181.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T21:32:29.099 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================== 2024-09-13T21:32:29.099 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:32:29.099 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:32:29.099 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:32:29.099 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T21:32:29.100 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T21:32:29.196 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'ecab7be58963a5bd03910cd73eef66405d7fdd50'} 2024-09-13T21:32:29.196 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-13T21:32:29.196 INFO:tasks.cephadm:Cluster fsid is ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:32:29.196 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T21:32:29.197 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-13T21:32:29.197 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi154': '172.21.15.154', 'mon.smithi181': '172.21.15.181'} 2024-09-13T21:32:29.197 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T21:32:29.197 DEBUG:teuthology.orchestra.run.smithi154:> sudo hostname $(hostname -s) 2024-09-13T21:32:29.224 DEBUG:teuthology.orchestra.run.smithi181:> sudo hostname $(hostname -s) 2024-09-13T21:32:29.249 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-13T21:32:29.250 DEBUG:teuthology.orchestra.run.smithi154:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T21:32:29.504 INFO:teuthology.orchestra.run.smithi154.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 21:32 /home/ubuntu/cephtest/cephadm 2024-09-13T21:32:29.505 DEBUG:teuthology.orchestra.run.smithi181:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T21:32:29.632 INFO:teuthology.orchestra.run.smithi181.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 21:32 /home/ubuntu/cephtest/cephadm 2024-09-13T21:32:29.633 DEBUG:teuthology.orchestra.run.smithi154:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T21:32:29.650 DEBUG:teuthology.orchestra.run.smithi181:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T21:32:29.679 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-13T21:32:29.680 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T21:32:29.698 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T21:32:31.323 INFO:teuthology.orchestra.run.smithi154.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T21:32:31.365 INFO:teuthology.orchestra.run.smithi181.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T21:32:43.428 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2024-09-13T21:32:43.428 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T21:32:43.429 INFO:teuthology.orchestra.run.smithi181.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T21:32:43.429 INFO:teuthology.orchestra.run.smithi181.stdout: "repo_digests": [ 2024-09-13T21:32:43.429 INFO:teuthology.orchestra.run.smithi181.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T21:32:43.429 INFO:teuthology.orchestra.run.smithi181.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T21:32:43.429 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2024-09-13T21:32:43.429 INFO:teuthology.orchestra.run.smithi181.stdout:} 2024-09-13T21:32:44.084 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:32:44.085 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T21:32:44.085 INFO:teuthology.orchestra.run.smithi154.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T21:32:44.085 INFO:teuthology.orchestra.run.smithi154.stdout: "repo_digests": [ 2024-09-13T21:32:44.085 INFO:teuthology.orchestra.run.smithi154.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T21:32:44.085 INFO:teuthology.orchestra.run.smithi154.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T21:32:44.085 INFO:teuthology.orchestra.run.smithi154.stdout: ] 2024-09-13T21:32:44.085 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:32:44.104 DEBUG:teuthology.orchestra.run.smithi154:> sudo mkdir -p /etc/ceph 2024-09-13T21:32:44.161 DEBUG:teuthology.orchestra.run.smithi181:> sudo mkdir -p /etc/ceph 2024-09-13T21:32:44.191 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod 777 /etc/ceph 2024-09-13T21:32:44.216 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 777 /etc/ceph 2024-09-13T21:32:44.257 INFO:tasks.cephadm:Writing seed config... 2024-09-13T21:32:44.258 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T21:32:44.258 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T21:32:44.258 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T21:32:44.258 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T21:32:44.259 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T21:32:44.259 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T21:32:44.259 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T21:32:44.259 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T21:32:44.259 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:32:44.259 DEBUG:teuthology.orchestra.run.smithi154:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T21:32:44.274 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ad78d70a-7217-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T21:32:44.275 DEBUG:teuthology.orchestra.run.smithi154:mon.smithi154> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154.service 2024-09-13T21:32:44.317 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T21:32:44.317 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid ad78d70a-7217-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.154 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T21:32:44.506 INFO:teuthology.orchestra.run.smithi154.stderr:-------------------------------------------------------------------------------- 2024-09-13T21:32:44.506 INFO:teuthology.orchestra.run.smithi154.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'ad78d70a-7217-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.154', '--skip-admin-label'] 2024-09-13T21:32:44.529 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: 5.2.2 2024-09-13T21:32:44.531 INFO:teuthology.orchestra.run.smithi154.stderr:Verifying podman|docker is present... 2024-09-13T21:32:44.552 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: 5.2.2 2024-09-13T21:32:44.554 INFO:teuthology.orchestra.run.smithi154.stderr:Verifying lvm2 is present... 2024-09-13T21:32:44.554 INFO:teuthology.orchestra.run.smithi154.stderr:Verifying time synchronization is in place... 2024-09-13T21:32:44.561 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T21:32:44.567 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: inactive 2024-09-13T21:32:44.574 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: enabled 2024-09-13T21:32:44.580 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: active 2024-09-13T21:32:44.581 INFO:teuthology.orchestra.run.smithi154.stderr:Unit chronyd.service is enabled and running 2024-09-13T21:32:44.581 INFO:teuthology.orchestra.run.smithi154.stderr:Repeating the final host check... 2024-09-13T21:32:44.601 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: 5.2.2 2024-09-13T21:32:44.603 INFO:teuthology.orchestra.run.smithi154.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-13T21:32:44.603 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl is present 2024-09-13T21:32:44.603 INFO:teuthology.orchestra.run.smithi154.stderr:lvcreate is present 2024-09-13T21:32:44.609 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T21:32:44.615 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: inactive 2024-09-13T21:32:44.622 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: enabled 2024-09-13T21:32:44.628 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: active 2024-09-13T21:32:44.629 INFO:teuthology.orchestra.run.smithi154.stderr:Unit chronyd.service is enabled and running 2024-09-13T21:32:44.629 INFO:teuthology.orchestra.run.smithi154.stderr:Host looks OK 2024-09-13T21:32:44.629 INFO:teuthology.orchestra.run.smithi154.stderr:Cluster fsid: ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:32:44.629 INFO:teuthology.orchestra.run.smithi154.stderr:Acquiring lock 140384469321616 on /run/cephadm/ad78d70a-7217-11ef-bceb-c7b262605968.lock 2024-09-13T21:32:44.629 INFO:teuthology.orchestra.run.smithi154.stderr:Lock 140384469321616 acquired on /run/cephadm/ad78d70a-7217-11ef-bceb-c7b262605968.lock 2024-09-13T21:32:44.630 INFO:teuthology.orchestra.run.smithi154.stderr:Verifying IP 172.21.15.154 port 3300 ... 2024-09-13T21:32:44.630 INFO:teuthology.orchestra.run.smithi154.stderr:Verifying IP 172.21.15.154 port 6789 ... 2024-09-13T21:32:44.630 INFO:teuthology.orchestra.run.smithi154.stderr:Base mon IP is 172.21.15.154, final addrv is [v2:172.21.15.154:3300,v1:172.21.15.154:6789] 2024-09-13T21:32:44.632 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.154 metric 100 2024-09-13T21:32:44.632 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.154 metric 100 2024-09-13T21:32:44.635 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T21:32:44.635 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-13T21:32:44.635 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-13T21:32:44.637 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T21:32:44.638 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-13T21:32:44.638 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T21:32:44.638 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-13T21:32:44.638 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:7761/64 scope link noprefixroute 2024-09-13T21:32:44.638 INFO:teuthology.orchestra.run.smithi154.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T21:32:44.638 INFO:teuthology.orchestra.run.smithi154.stderr:Mon IP `172.21.15.154` is in CIDR network `172.21.0.0/20` 2024-09-13T21:32:44.638 INFO:teuthology.orchestra.run.smithi154.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T21:32:44.639 INFO:teuthology.orchestra.run.smithi154.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T21:32:44.737 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-13T21:32:45.318 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Getting image source signatures 2024-09-13T21:32:45.318 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-13T21:32:45.318 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-13T21:32:45.318 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-13T21:32:45.318 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-13T21:32:45.318 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-13T21:32:45.319 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T21:32:45.497 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: Writing manifest to image destination 2024-09-13T21:32:45.649 INFO:teuthology.orchestra.run.smithi154.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T21:32:46.059 INFO:teuthology.orchestra.run.smithi154.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T21:32:46.623 INFO:teuthology.orchestra.run.smithi154.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T21:32:46.623 INFO:teuthology.orchestra.run.smithi154.stderr:Extracting ceph user uid/gid from container image... 2024-09-13T21:32:47.348 INFO:teuthology.orchestra.run.smithi154.stderr:stat: 167 167 2024-09-13T21:32:47.845 INFO:teuthology.orchestra.run.smithi154.stderr:Creating initial keys... 2024-09-13T21:32:48.211 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph-authtool: AQCAr+RmMtHzCRAA61rvnLENysVJALEWXMX0/A== 2024-09-13T21:32:49.158 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph-authtool: AQCBr+RmLfzJBBAA/ABN1lWHXWyaRyJdx0NfGw== 2024-09-13T21:32:50.189 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph-authtool: AQCCr+Rmqx8SBhAAOdJqw78t+kTHcDAu2pw5lw== 2024-09-13T21:32:50.704 INFO:teuthology.orchestra.run.smithi154.stderr:Creating initial monmap... 2024-09-13T21:32:51.069 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T21:32:51.069 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-13T21:32:51.070 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:32:51.070 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T21:32:51.575 INFO:teuthology.orchestra.run.smithi154.stderr:monmaptool for smithi154 [v2:172.21.15.154:3300,v1:172.21.15.154:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T21:32:51.575 INFO:teuthology.orchestra.run.smithi154.stderr:setting min_mon_release = octopus 2024-09-13T21:32:51.575 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/monmaptool: set fsid to ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:32:51.575 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T21:32:51.575 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:32:51.575 INFO:teuthology.orchestra.run.smithi154.stderr:Creating mon... 2024-09-13T21:32:58.450 INFO:teuthology.orchestra.run.smithi154.stderr:create mon.smithi154 on 2024-09-13T21:32:58.744 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T21:32:59.015 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ad78d70a-7217-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-ad78d70a-7217-11ef-bceb-c7b262605968.target. 2024-09-13T21:32:59.016 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-ad78d70a-7217-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-ad78d70a-7217-11ef-bceb-c7b262605968.target. 2024-09-13T21:32:59.732 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Failed to reset failed state of unit ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154.service: Unit ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154.service not loaded. 2024-09-13T21:32:59.740 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ad78d70a-7217-11ef-bceb-c7b262605968.target.wants/ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154.service → /etc/systemd/system/ceph-ad78d70a-7217-11ef-bceb-c7b262605968@.service. 2024-09-13T21:33:00.657 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: disabled 2024-09-13T21:33:00.663 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: inactive 2024-09-13T21:33:00.664 INFO:teuthology.orchestra.run.smithi154.stderr:firewalld.service is not enabled 2024-09-13T21:33:00.664 INFO:teuthology.orchestra.run.smithi154.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T21:33:00.664 INFO:teuthology.orchestra.run.smithi154.stderr:Waiting for mon to start... 2024-09-13T21:33:00.665 INFO:teuthology.orchestra.run.smithi154.stderr:Waiting for mon... 2024-09-13T21:33:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:00 smithi154 systemd[1]: Started Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:33:01.139 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: cluster: 2024-09-13T21:33:01.139 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: id: ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:33:01.139 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-13T21:33:01.139 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:01.139 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: services: 2024-09-13T21:33:01.139 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi154 (age 0.566983s) 2024-09-13T21:33:01.139 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-13T21:33:01.139 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-13T21:33:01.140 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:01.140 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: data: 2024-09-13T21:33:01.140 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-13T21:33:01.140 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-13T21:33:01.140 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-13T21:33:01.140 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: pgs: 2024-09-13T21:33:01.140 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:01.693 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:01 smithi154 ceph-mon[23360]: mon.smithi154 is new leader, mons smithi154 in quorum (ranks 0) 2024-09-13T21:33:01.693 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:01 smithi154 ceph-mon[23360]: monmap e1: 1 mons at {smithi154=[v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0]} 2024-09-13T21:33:01.693 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:01 smithi154 ceph-mon[23360]: fsmap 2024-09-13T21:33:01.693 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:01 smithi154 ceph-mon[23360]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T21:33:01.693 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:01 smithi154 ceph-mon[23360]: mgrmap e1: no daemons active 2024-09-13T21:33:01.693 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:01 smithi154 ceph-mon[23360]: from='client.? 172.21.15.154:0/1932289113' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T21:33:01.695 INFO:teuthology.orchestra.run.smithi154.stderr:mon is available 2024-09-13T21:33:01.695 INFO:teuthology.orchestra.run.smithi154.stderr:Assimilating anything we can from ceph.conf... 2024-09-13T21:33:02.177 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:02.177 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: [global] 2024-09-13T21:33:02.177 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: fsid = ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:33:02.177 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.154:3300,v1:172.21.15.154:6789] 2024-09-13T21:33:02.177 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T21:33:02.177 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: [mgr] 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: [osd] 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T21:33:02.178 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T21:33:02.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:02 smithi154 ceph-mon[23360]: from='client.? 172.21.15.154:0/1516773804' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T21:33:02.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:02 smithi154 ceph-mon[23360]: from='client.? 172.21.15.154:0/1516773804' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T21:33:02.768 INFO:teuthology.orchestra.run.smithi154.stderr:Generating new minimal ceph.conf... 2024-09-13T21:33:03.797 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:03 smithi154 ceph-mon[23360]: from='client.? 172.21.15.154:0/933604871' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:33:03.799 INFO:teuthology.orchestra.run.smithi154.stderr:Restarting the monitor... 2024-09-13T21:33:04.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:03 smithi154 systemd[1]: Stopping Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:33:04.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:03 smithi154 bash[23579]: Error: no container with name or ID "ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon.smithi154" found: no such container 2024-09-13T21:33:04.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:03 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[23356]: 2024-09-13T21:33:03.956+0000 7fedc0c4e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi154 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:33:04.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:03 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[23356]: 2024-09-13T21:33:03.956+0000 7fedc0c4e700 -1 mon.smithi154@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T21:33:04.391 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 podman[23590]: 2024-09-13 21:33:04.089726154 +0000 UTC m=+0.236742477 container died 1f8efd1524c84b9060230eb143f9c46bd309b3579587827a816c94611a84d500 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_CLEAN=True, version=8, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, name=centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, maintainer=Guillaume Abrioux , vcs-type=git, io.buildah.version=1.19.8) 2024-09-13T21:33:04.391 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 podman[23590]: 2024-09-13 21:33:04.389549561 +0000 UTC m=+0.536565883 container cleanup 1f8efd1524c84b9060230eb143f9c46bd309b3579587827a816c94611a84d500 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, GIT_BRANCH=HEAD, name=centos-stream, distribution-scope=public, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.expose-services=, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8) 2024-09-13T21:33:04.814 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 bash[23590]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154 2024-09-13T21:33:04.815 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 bash[23618]: Error: no container with name or ID "ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon.smithi154" found: no such container 2024-09-13T21:33:04.815 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 podman[23602]: 2024-09-13 21:33:04.590499898 +0000 UTC m=+0.500105624 container remove 1f8efd1524c84b9060230eb143f9c46bd309b3579587827a816c94611a84d500 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, version=8) 2024-09-13T21:33:04.815 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154.service: Deactivated successfully. 2024-09-13T21:33:04.815 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 systemd[1]: Stopped Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:33:04.815 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 systemd[1]: Starting Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 podman[23682]: 2024-09-13 21:33:04.716547001 +0000 UTC m=+0.016385602 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:04 smithi154 podman[23682]: 2024-09-13 21:33:04.824443517 +0000 UTC m=+0.124282164 container create ca544568ebc042e7dcd70d02923dfc3b9d56f779e7a4cef0931bce03429266ad (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, version=8, ceph=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 podman[23682]: 2024-09-13 21:33:05.02720257 +0000 UTC m=+0.327041180 container init ca544568ebc042e7dcd70d02923dfc3b9d56f779e7a4cef0931bce03429266ad (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, ceph=True, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, version=8, name=centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-type=git, architecture=x86_64) 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 podman[23682]: 2024-09-13 21:33:05.030536196 +0000 UTC m=+0.330374799 container start ca544568ebc042e7dcd70d02923dfc3b9d56f779e7a4cef0931bce03429266ad (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., RELEASE=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux ) 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: pidfile_write: ignore empty --pid-file 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: load: jerasure load: lrc 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: RocksDB version: 6.15.5 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T21:33:05.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Compile date Apr 18 2022 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: DB SUMMARY 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: DB Session ID: MSQ1ZWMMKGM1I6GWQ6U1 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: CURRENT file: CURRENT 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi154/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi154/store.db: 000010.log size: 82737 ; 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.error_if_exists: 0 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.create_if_missing: 0 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.paranoid_checks: 1 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.env: 0x55d3dc2c6860 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.fs: Posix File System 2024-09-13T21:33:05.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.info_log: 0x55d3dd057fa0 2024-09-13T21:33:05.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T21:33:05.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.statistics: (nil) 2024-09-13T21:33:05.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.use_fsync: 0 2024-09-13T21:33:05.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_log_file_size: 0 2024-09-13T21:33:05.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T21:33:05.093 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T21:33:05.093 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T21:33:05.093 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T21:33:05.093 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.allow_fallocate: 1 2024-09-13T21:33:05.093 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.use_direct_reads: 0 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.db_log_dir: 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi154/store.db 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T21:33:05.094 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.write_buffer_manager: 0x55d3dd1482d0 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.unordered_write: 0 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T21:33:05.095 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.row_cache: None 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.wal_filter: None 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.preserve_deletes: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.two_write_queues: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.atomic_flush: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T21:33:05.096 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.log_readahead_size: 0 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_background_jobs: 2 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_background_compactions: -1 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_subcompactions: 1 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T21:33:05.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_open_files: -1 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_background_flushes: -1 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Compression algorithms supported: 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: kZSTD supported: 0 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: kXpressCompression supported: 0 2024-09-13T21:33:05.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: kLZ4Compression supported: 1 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: kBZip2Compression supported: 0 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: kZlibCompression supported: 1 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: kSnappyCompression supported: 1 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi154/store.db/MANIFEST-000009 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.merge_operator: 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_filter: None 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T21:33:05.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d3dd023d00) 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: cache_index_and_filter_blocks: 1 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: pin_top_level_index_and_filter: 1 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_type: 0 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: data_block_index_type: 0 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_shortening: 1 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: hash_index_allow_collision: 1 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: checksum: 1 2024-09-13T21:33:05.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: no_block_cache: 0 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache: 0x55d3dd08e170 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_name: BinnedLRUCache 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_options: 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: capacity : 536870912 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: num_shard_bits : 4 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: strict_capacity_limit : 0 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: high_pri_pool_ratio: 0.000 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_compressed: (nil) 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: persistent_cache: (nil) 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_size: 4096 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_size_deviation: 10 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_restart_interval: 16 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_block_restart_interval: 1 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: metadata_block_size: 4096 2024-09-13T21:33:05.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: partition_filters: 0 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: use_delta_encoding: 1 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: whole_key_filtering: 1 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: verify_compression: 0 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: read_amp_bytes_per_bit: 0 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: format_version: 4 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: enable_index_compression: 1 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_align: 0 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compression: NoCompression 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.num_levels: 7 2024-09-13T21:33:05.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:33:05.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T21:33:05.104 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T21:33:05.105 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.table_properties_collectors: 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.inplace_update_support: 0 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.bloom_locality: 0 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.max_successive_merges: 0 2024-09-13T21:33:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.ttl: 2592000 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.enable_blob_files: false 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.min_blob_size: 0 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi154/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T21:33:05.107 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263185056492, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263185057224, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79419, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77683, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71992, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726263185, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "3e873461-9702-4c19-a304-7156db38235b", "db_session_id": "MSQ1ZWMMKGM1I6GWQ6U1"}} 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263185057551, "job": 1, "event": "recovery_finished"} 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi154/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55d3dd074a80 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: DB pointer 0x55d3dd084000 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** DB Stats ** 2024-09-13T21:33:05.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 150.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 150.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:33:05.109 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 150.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 150.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative compaction: 0.00 GB write, 24.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval compaction: 0.00 GB write, 24.89 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:33:05.110 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 150.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 150.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 150.9 0.00 0.00 1 0.001 0 0 2024-09-13T21:33:05.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative compaction: 0.00 GB write, 24.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: starting mon.smithi154 rank 0 at public addrs [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] at bind addrs [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi154 fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154@-1(???) e1 preinit fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154@-1(???).mds e1 new map 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154@-1(???).mds e1 print_map 2024-09-13T21:33:05.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: e1 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: legacy client fscid: -1 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: No filesystems configured 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:33:05.113 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T21:33:05.117 INFO:teuthology.orchestra.run.smithi154.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-13T21:33:05.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 bash[23682]: ca544568ebc042e7dcd70d02923dfc3b9d56f779e7a4cef0931bce03429266ad 2024-09-13T21:33:05.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 systemd[1]: Started Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:33:05.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mon.smithi154 is new leader, mons smithi154 in quorum (ranks 0) 2024-09-13T21:33:05.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: monmap e1: 1 mons at {smithi154=[v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0]} 2024-09-13T21:33:05.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: fsmap 2024-09-13T21:33:05.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T21:33:05.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:05 smithi154 ceph-mon[23700]: mgrmap e1: no daemons active 2024-09-13T21:33:08.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:08 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3769813089' entity='client.admin' 2024-09-13T21:33:10.322 INFO:teuthology.orchestra.run.smithi154.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-13T21:33:10.322 INFO:teuthology.orchestra.run.smithi154.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T21:33:10.322 INFO:teuthology.orchestra.run.smithi154.stderr:Creating mgr... 2024-09-13T21:33:10.322 INFO:teuthology.orchestra.run.smithi154.stderr:Verifying port 9283 ... 2024-09-13T21:33:10.501 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Failed to reset failed state of unit ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mgr.smithi154.ayrwdi.service: Unit ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mgr.smithi154.ayrwdi.service not loaded. 2024-09-13T21:33:10.508 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ad78d70a-7217-11ef-bceb-c7b262605968.target.wants/ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mgr.smithi154.ayrwdi.service → /etc/systemd/system/ceph-ad78d70a-7217-11ef-bceb-c7b262605968@.service. 2024-09-13T21:33:11.176 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: disabled 2024-09-13T21:33:11.185 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: inactive 2024-09-13T21:33:11.185 INFO:teuthology.orchestra.run.smithi154.stderr:firewalld.service is not enabled 2024-09-13T21:33:11.186 INFO:teuthology.orchestra.run.smithi154.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T21:33:11.194 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: disabled 2024-09-13T21:33:11.202 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: inactive 2024-09-13T21:33:11.203 INFO:teuthology.orchestra.run.smithi154.stderr:firewalld.service is not enabled 2024-09-13T21:33:11.203 INFO:teuthology.orchestra.run.smithi154.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-13T21:33:11.203 INFO:teuthology.orchestra.run.smithi154.stderr:Waiting for mgr to start... 2024-09-13T21:33:11.203 INFO:teuthology.orchestra.run.smithi154.stderr:Waiting for mgr... 2024-09-13T21:33:12.340 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:12.340 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: { 2024-09-13T21:33:12.340 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "fsid": "ad78d70a-7217-11ef-bceb-c7b262605968", 2024-09-13T21:33:12.340 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "health": { 2024-09-13T21:33:12.340 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T21:33:12.340 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 0 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "smithi154" 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum_age": 7, 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T21:33:12.341 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T21:33:12.342 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:12.342 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T21:33:12.342 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:12.342 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T21:33:12.342 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T21:33:12.342 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T21:33:12.342 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T21:33:12.345 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T21:33:12.345 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T21:33:12.346 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "available": false, 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "iostat", 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "nfs", 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "restful" 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "modified": "2024-09-13T21:33:00.570535+0000", 2024-09-13T21:33:12.347 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:33:12.348 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:12.348 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T21:33:12.348 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: } 2024-09-13T21:33:12.501 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:12 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3916170133' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T21:33:12.948 INFO:teuthology.orchestra.run.smithi154.stderr:mgr not available, waiting (1/15)... 2024-09-13T21:33:15.521 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:15.521 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: { 2024-09-13T21:33:15.521 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "fsid": "ad78d70a-7217-11ef-bceb-c7b262605968", 2024-09-13T21:33:15.521 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "health": { 2024-09-13T21:33:15.521 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 0 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "smithi154" 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum_age": 10, 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T21:33:15.522 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T21:33:15.523 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T21:33:15.526 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T21:33:15.526 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:15.527 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "available": false, 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "iostat", 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "nfs", 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "restful" 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T21:33:15.528 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:15.529 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "modified": "2024-09-13T21:33:00.570535+0000", 2024-09-13T21:33:15.529 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:33:15.529 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:15.529 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T21:33:15.529 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: } 2024-09-13T21:33:16.163 INFO:teuthology.orchestra.run.smithi154.stderr:mgr not available, waiting (2/15)... 2024-09-13T21:33:16.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:16 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3423939083' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T21:33:17.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: Activating manager daemon smithi154.ayrwdi 2024-09-13T21:33:17.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: mgrmap e2: smithi154.ayrwdi(active, starting, since 0.00265839s) 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:17.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:17 smithi154 ceph-mon[23700]: from='mgr.14100 172.21.15.154:0/3332989460' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:18.856 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:18.857 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: { 2024-09-13T21:33:18.857 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "fsid": "ad78d70a-7217-11ef-bceb-c7b262605968", 2024-09-13T21:33:18.857 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "health": { 2024-09-13T21:33:18.857 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T21:33:18.857 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T21:33:18.857 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T21:33:18.857 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:18.857 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T21:33:18.858 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T21:33:18.858 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 0 2024-09-13T21:33:18.858 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:18.858 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T21:33:18.858 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "smithi154" 2024-09-13T21:33:18.858 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:18.858 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "quorum_age": 13, 2024-09-13T21:33:18.859 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T21:33:18.859 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:18.859 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T21:33:18.859 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T21:33:18.860 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T21:33:18.861 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "available": true, 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "iostat", 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "nfs", 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "restful" 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ], 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T21:33:18.862 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "modified": "2024-09-13T21:33:00.570535+0000", 2024-09-13T21:33:18.863 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "services": {} 2024-09-13T21:33:18.863 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: }, 2024-09-13T21:33:18.863 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T21:33:18.863 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: } 2024-09-13T21:33:19.474 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:19 smithi154 ceph-mon[23700]: mgrmap e3: smithi154.ayrwdi(active, since 1.00663s) 2024-09-13T21:33:19.475 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:19 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3346531167' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T21:33:19.477 INFO:teuthology.orchestra.run.smithi154.stderr:mgr is available 2024-09-13T21:33:20.105 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:20.105 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: [global] 2024-09-13T21:33:20.105 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: fsid = ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:33:20.105 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T21:33:20.105 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T21:33:20.105 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T21:33:20.106 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T21:33:20.106 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:20.106 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: [mgr] 2024-09-13T21:33:20.106 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T21:33:20.106 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 2024-09-13T21:33:20.106 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: [osd] 2024-09-13T21:33:20.106 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T21:33:20.106 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T21:33:20.396 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:20 smithi154 ceph-mon[23700]: mgrmap e4: smithi154.ayrwdi(active, since 2s) 2024-09-13T21:33:20.397 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:20 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/4004719800' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T21:33:20.685 INFO:teuthology.orchestra.run.smithi154.stderr:Enabling cephadm module... 2024-09-13T21:33:21.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:21 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/269385392' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T21:33:22.510 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:22 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/269385392' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T21:33:22.510 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:22 smithi154 ceph-mon[23700]: mgrmap e5: smithi154.ayrwdi(active, since 4s) 2024-09-13T21:33:22.642 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: { 2024-09-13T21:33:22.642 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-13T21:33:22.643 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "available": true, 2024-09-13T21:33:22.643 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "active_name": "smithi154.ayrwdi", 2024-09-13T21:33:22.643 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T21:33:22.643 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: } 2024-09-13T21:33:23.179 INFO:teuthology.orchestra.run.smithi154.stderr:Waiting for the mgr to restart... 2024-09-13T21:33:23.180 INFO:teuthology.orchestra.run.smithi154.stderr:Waiting for mgr epoch 5... 2024-09-13T21:33:23.430 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:23 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/330038966' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T21:33:28.499 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: { 2024-09-13T21:33:28.499 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-13T21:33:28.499 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T21:33:28.499 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: } 2024-09-13T21:33:28.806 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: Active manager daemon smithi154.ayrwdi restarted 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: Activating manager daemon smithi154.ayrwdi 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: mgrmap e6: smithi154.ayrwdi(active, starting, since 0.0524418s) 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:28.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:33:28.808 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:33:28.808 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:33:28.808 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:33:28.808 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:28 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:33:29.035 INFO:teuthology.orchestra.run.smithi154.stderr:mgr epoch 5 is available 2024-09-13T21:33:29.035 INFO:teuthology.orchestra.run.smithi154.stderr:Setting orchestrator backend to cephadm... 2024-09-13T21:33:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:29 smithi154 ceph-mon[23700]: mgrmap e7: smithi154.ayrwdi(active, since 1.05486s) 2024-09-13T21:33:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:29 smithi154 ceph-mon[23700]: [13/Sep/2024:21:33:28] ENGINE Bus STARTING 2024-09-13T21:33:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:29 smithi154 ceph-mon[23700]: [13/Sep/2024:21:33:29] ENGINE Serving on https://172.21.15.154:7150 2024-09-13T21:33:29.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:29 smithi154 ceph-mon[23700]: [13/Sep/2024:21:33:29] ENGINE Bus STARTED 2024-09-13T21:33:29.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:29 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:29.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:29 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:33:30.884 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: value unchanged 2024-09-13T21:33:31.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:30 smithi154 ceph-mon[23700]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:31.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:30 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:31.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:30 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:33:31.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:30 smithi154 ceph-mon[23700]: mgrmap e8: smithi154.ayrwdi(active, since 2s) 2024-09-13T21:33:31.396 INFO:teuthology.orchestra.run.smithi154.stderr:Generating ssh key... 2024-09-13T21:33:32.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:31 smithi154 ceph-mon[23700]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:32.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:32 smithi154 ceph-mon[23700]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:32.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:32 smithi154 ceph-mon[23700]: Generating ssh key... 2024-09-13T21:33:32.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:32 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:32.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:32 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:33.546 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSam/wBLRVR6D/RY8QizX+F0PIuH5sEJLFMZR+wI72dzz1x1CSWD9pzwEP6xBoSohXlNw2wuWPFuAXHmMGzEldp4+BBq02uKiatKGV0caKzI4ss+/ufk8JrwIOpU7ZaWahgt/z8ROB0JhstqbOjTyZay9Y2wPTlAi3lSHzO5+RR1NP6+ACTiwYq/zu6ynNNjp+feDpNRXJ3aqVWFOxC9l2s8ePeKVhXr6QElwr/3vcxh9pRs0K+b+4TnoiTzscXD3M7ztlpwr1ZXguUJ7Jh2ORDMHhU1LHZU52ZPtRISIg54iVrF4Udal4/hGhLMYZQ2fZTo4Y1Z631qf07JSROoESGkZF7+gjuBjxnWtexYw83858OgimxKZVHW3BbcLzFJXGcpG7wCNxRLDWTPPwj0yQ8v7PgaU/v5ueowkhnqVsJMjSDYicoPvb2VJE6w71pjqOC0uuHoIk8EpekS1iLdhBnSFe5GeYuxI7AphMO4GTZJpE+IoaF/K2V6jBjrxKwVk= ceph-ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:33:34.084 INFO:teuthology.orchestra.run.smithi154.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T21:33:34.084 INFO:teuthology.orchestra.run.smithi154.stderr:Adding key to root@localhost authorized_keys... 2024-09-13T21:33:34.090 INFO:teuthology.orchestra.run.smithi154.stderr:Adding host smithi154... 2024-09-13T21:33:34.729 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:34 smithi154 ceph-mon[23700]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:35.566 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Added host 'smithi154' with addr '172.21.15.154' 2024-09-13T21:33:36.222 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:36 smithi154 ceph-mon[23700]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi154", "addr": "172.21.15.154", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:36.222 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:36 smithi154 ceph-mon[23700]: Deploying cephadm binary to smithi154 2024-09-13T21:33:36.222 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:36 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:36.222 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:36 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:33:36.224 INFO:teuthology.orchestra.run.smithi154.stderr:Deploying mon service with default placement... 2024-09-13T21:33:36.853 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-13T21:33:37.197 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:37 smithi154 ceph-mon[23700]: Added host smithi154 2024-09-13T21:33:37.197 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:37 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:37.584 INFO:teuthology.orchestra.run.smithi154.stderr:Deploying mgr service with default placement... 2024-09-13T21:33:38.175 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:38 smithi154 ceph-mon[23700]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:38.175 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:38 smithi154 ceph-mon[23700]: Saving service mon spec with placement count:5 2024-09-13T21:33:38.176 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-13T21:33:38.725 INFO:teuthology.orchestra.run.smithi154.stderr:Deploying crash service with default placement... 2024-09-13T21:33:39.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:39 smithi154 ceph-mon[23700]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:39.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:39 smithi154 ceph-mon[23700]: Saving service mgr spec with placement count:2 2024-09-13T21:33:39.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:39 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:40.091 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-13T21:33:40.677 INFO:teuthology.orchestra.run.smithi154.stderr:Deploying prometheus service with default placement... 2024-09-13T21:33:40.944 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:40 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:40.944 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:40 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:40.944 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:40 smithi154 ceph-mon[23700]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:40.944 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:40 smithi154 ceph-mon[23700]: Saving service crash spec with placement * 2024-09-13T21:33:40.944 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:40 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:41.245 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-13T21:33:42.386 INFO:teuthology.orchestra.run.smithi154.stderr:Deploying grafana service with default placement... 2024-09-13T21:33:42.508 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:42 smithi154 ceph-mon[23700]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:42.509 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:42 smithi154 ceph-mon[23700]: Saving service prometheus spec with placement count:1 2024-09-13T21:33:42.509 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:42 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:42.509 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:42 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:43.098 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-13T21:33:43.652 INFO:teuthology.orchestra.run.smithi154.stderr:Deploying node-exporter service with default placement... 2024-09-13T21:33:44.261 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:43 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:44.262 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:43 smithi154 ceph-mon[23700]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:44.262 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:43 smithi154 ceph-mon[23700]: Saving service grafana spec with placement count:1 2024-09-13T21:33:44.262 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:43 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:44.274 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-13T21:33:44.870 INFO:teuthology.orchestra.run.smithi154.stderr:Deploying alertmanager service with default placement... 2024-09-13T21:33:45.489 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:45 smithi154 ceph-mon[23700]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:45.489 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:45 smithi154 ceph-mon[23700]: Saving service node-exporter spec with placement * 2024-09-13T21:33:45.489 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:45 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:45.489 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-13T21:33:46.836 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:46 smithi154 ceph-mon[23700]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:46.836 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:46 smithi154 ceph-mon[23700]: Saving service alertmanager spec with placement count:1 2024-09-13T21:33:46.836 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:46 smithi154 ceph-mon[23700]: from='mgr.14118 172.21.15.154:0/766700689' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:48.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:47 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/519168358' entity='client.admin' 2024-09-13T21:33:48.753 INFO:teuthology.orchestra.run.smithi154.stderr:Enabling the dashboard module... 2024-09-13T21:33:49.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:49 smithi154 ceph-mon[23700]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:33:49.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:49 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3149291247' entity='client.admin' 2024-09-13T21:33:50.559 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:50 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2144647162' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T21:33:51.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:51 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2144647162' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T21:33:51.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:51 smithi154 ceph-mon[23700]: mgrmap e9: smithi154.ayrwdi(active, since 22s) 2024-09-13T21:33:51.529 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: { 2024-09-13T21:33:51.530 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-13T21:33:51.530 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "available": true, 2024-09-13T21:33:51.530 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "active_name": "smithi154.ayrwdi", 2024-09-13T21:33:51.530 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T21:33:51.530 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: } 2024-09-13T21:33:52.054 INFO:teuthology.orchestra.run.smithi154.stderr:Waiting for the mgr to restart... 2024-09-13T21:33:52.054 INFO:teuthology.orchestra.run.smithi154.stderr:Waiting for mgr epoch 9... 2024-09-13T21:33:52.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:52 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3790287533' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T21:33:56.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:56 smithi154 ceph-mon[23700]: Active manager daemon smithi154.ayrwdi restarted 2024-09-13T21:33:56.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:56 smithi154 ceph-mon[23700]: Activating manager daemon smithi154.ayrwdi 2024-09-13T21:33:56.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:56 smithi154 ceph-mon[23700]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T21:33:57.468 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: { 2024-09-13T21:33:57.468 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-13T21:33:57.468 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T21:33:57.468 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: } 2024-09-13T21:33:57.817 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: mgrmap e10: smithi154.ayrwdi(active, starting, since 0.0554192s) 2024-09-13T21:33:57.817 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:33:57.817 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:33:57.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:33:57.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:33:57.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:33:57.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:33:57.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:33:57.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:33:57.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:33:57.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:57 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:33:58.129 INFO:teuthology.orchestra.run.smithi154.stderr:mgr epoch 9 is available 2024-09-13T21:33:58.129 INFO:teuthology.orchestra.run.smithi154.stderr:Generating a dashboard self-signed certificate... 2024-09-13T21:33:58.696 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:58 smithi154 ceph-mon[23700]: mgrmap e11: smithi154.ayrwdi(active, since 1.06544s) 2024-09-13T21:33:58.696 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:58 smithi154 ceph-mon[23700]: [13/Sep/2024:21:33:57] ENGINE Bus STARTING 2024-09-13T21:33:58.696 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:58 smithi154 ceph-mon[23700]: [13/Sep/2024:21:33:57] ENGINE Serving on https://172.21.15.154:7150 2024-09-13T21:33:58.696 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:58 smithi154 ceph-mon[23700]: [13/Sep/2024:21:33:57] ENGINE Bus STARTED 2024-09-13T21:33:58.696 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:58 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:58.696 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:58 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:59.013 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-13T21:33:59.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:59 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:59.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:59 smithi154 ceph-mon[23700]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:33:59.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:59 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:59.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:59 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:33:59.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:33:59 smithi154 ceph-mon[23700]: mgrmap e12: smithi154.ayrwdi(active, since 2s) 2024-09-13T21:33:59.855 INFO:teuthology.orchestra.run.smithi154.stderr:Creating initial admin user... 2024-09-13T21:34:00.678 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$FYMGPlLxNQOjRWlQGfiUxuVyUT5MUCIsQpyMJHfrl3Fd1nAMcA.9W", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726263240, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T21:34:01.247 INFO:teuthology.orchestra.run.smithi154.stderr:Fetching dashboard port number... 2024-09-13T21:34:01.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:01 smithi154 ceph-mon[23700]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:34:01.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:01 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:01.842 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: 8443 2024-09-13T21:34:02.792 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: disabled 2024-09-13T21:34:02.799 INFO:teuthology.orchestra.run.smithi154.stderr:systemctl: inactive 2024-09-13T21:34:02.800 INFO:teuthology.orchestra.run.smithi154.stderr:firewalld.service is not enabled 2024-09-13T21:34:02.800 INFO:teuthology.orchestra.run.smithi154.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T21:34:02.802 INFO:teuthology.orchestra.run.smithi154.stderr:Ceph Dashboard is now available at: 2024-09-13T21:34:02.802 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:34:02.802 INFO:teuthology.orchestra.run.smithi154.stderr: URL: https://smithi154.front.sepia.ceph.com:8443/ 2024-09-13T21:34:02.802 INFO:teuthology.orchestra.run.smithi154.stderr: User: admin 2024-09-13T21:34:02.802 INFO:teuthology.orchestra.run.smithi154.stderr: Password: fnok2pgjro 2024-09-13T21:34:02.802 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:34:02.803 INFO:teuthology.orchestra.run.smithi154.stderr:Enabling autotune for osd_memory_target 2024-09-13T21:34:02.989 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:02 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2851726533' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T21:34:04.002 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:03 smithi154 ceph-mon[23700]: mgrmap e13: smithi154.ayrwdi(active, since 6s) 2024-09-13T21:34:04.740 INFO:teuthology.orchestra.run.smithi154.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-13T21:34:05.338 INFO:teuthology.orchestra.run.smithi154.stderr:You can access the Ceph CLI with: 2024-09-13T21:34:05.338 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:34:05.338 INFO:teuthology.orchestra.run.smithi154.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T21:34:05.338 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr: ceph telemetry on 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr:For more information see: 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T21:34:05.339 INFO:teuthology.orchestra.run.smithi154.stderr:Bootstrap complete. 2024-09-13T21:34:05.372 INFO:tasks.cephadm:Fetching config... 2024-09-13T21:34:05.372 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:34:05.373 DEBUG:teuthology.orchestra.run.smithi154:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T21:34:05.388 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T21:34:05.389 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:34:05.389 DEBUG:teuthology.orchestra.run.smithi154:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T21:34:05.445 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T21:34:05.445 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:34:05.446 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/keyring of=/dev/stdout 2024-09-13T21:34:05.511 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T21:34:05.511 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:34:05.511 DEBUG:teuthology.orchestra.run.smithi154:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T21:34:05.567 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T21:34:05.567 DEBUG:teuthology.orchestra.run.smithi154:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSam/wBLRVR6D/RY8QizX+F0PIuH5sEJLFMZR+wI72dzz1x1CSWD9pzwEP6xBoSohXlNw2wuWPFuAXHmMGzEldp4+BBq02uKiatKGV0caKzI4ss+/ufk8JrwIOpU7ZaWahgt/z8ROB0JhstqbOjTyZay9Y2wPTlAi3lSHzO5+RR1NP6+ACTiwYq/zu6ynNNjp+feDpNRXJ3aqVWFOxC9l2s8ePeKVhXr6QElwr/3vcxh9pRs0K+b+4TnoiTzscXD3M7ztlpwr1ZXguUJ7Jh2ORDMHhU1LHZU52ZPtRISIg54iVrF4Udal4/hGhLMYZQ2fZTo4Y1Z631qf07JSROoESGkZF7+gjuBjxnWtexYw83858OgimxKZVHW3BbcLzFJXGcpG7wCNxRLDWTPPwj0yQ8v7PgaU/v5ueowkhnqVsJMjSDYicoPvb2VJE6w71pjqOC0uuHoIk8EpekS1iLdhBnSFe5GeYuxI7AphMO4GTZJpE+IoaF/K2V6jBjrxKwVk= ceph-ad78d70a-7217-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T21:34:05.648 INFO:teuthology.orchestra.run.smithi154.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSam/wBLRVR6D/RY8QizX+F0PIuH5sEJLFMZR+wI72dzz1x1CSWD9pzwEP6xBoSohXlNw2wuWPFuAXHmMGzEldp4+BBq02uKiatKGV0caKzI4ss+/ufk8JrwIOpU7ZaWahgt/z8ROB0JhstqbOjTyZay9Y2wPTlAi3lSHzO5+RR1NP6+ACTiwYq/zu6ynNNjp+feDpNRXJ3aqVWFOxC9l2s8ePeKVhXr6QElwr/3vcxh9pRs0K+b+4TnoiTzscXD3M7ztlpwr1ZXguUJ7Jh2ORDMHhU1LHZU52ZPtRISIg54iVrF4Udal4/hGhLMYZQ2fZTo4Y1Z631qf07JSROoESGkZF7+gjuBjxnWtexYw83858OgimxKZVHW3BbcLzFJXGcpG7wCNxRLDWTPPwj0yQ8v7PgaU/v5ueowkhnqVsJMjSDYicoPvb2VJE6w71pjqOC0uuHoIk8EpekS1iLdhBnSFe5GeYuxI7AphMO4GTZJpE+IoaF/K2V6jBjrxKwVk= ceph-ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:34:05.660 DEBUG:teuthology.orchestra.run.smithi181:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSam/wBLRVR6D/RY8QizX+F0PIuH5sEJLFMZR+wI72dzz1x1CSWD9pzwEP6xBoSohXlNw2wuWPFuAXHmMGzEldp4+BBq02uKiatKGV0caKzI4ss+/ufk8JrwIOpU7ZaWahgt/z8ROB0JhstqbOjTyZay9Y2wPTlAi3lSHzO5+RR1NP6+ACTiwYq/zu6ynNNjp+feDpNRXJ3aqVWFOxC9l2s8ePeKVhXr6QElwr/3vcxh9pRs0K+b+4TnoiTzscXD3M7ztlpwr1ZXguUJ7Jh2ORDMHhU1LHZU52ZPtRISIg54iVrF4Udal4/hGhLMYZQ2fZTo4Y1Z631qf07JSROoESGkZF7+gjuBjxnWtexYw83858OgimxKZVHW3BbcLzFJXGcpG7wCNxRLDWTPPwj0yQ8v7PgaU/v5ueowkhnqVsJMjSDYicoPvb2VJE6w71pjqOC0uuHoIk8EpekS1iLdhBnSFe5GeYuxI7AphMO4GTZJpE+IoaF/K2V6jBjrxKwVk= ceph-ad78d70a-7217-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T21:34:05.807 INFO:teuthology.orchestra.run.smithi181.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDSam/wBLRVR6D/RY8QizX+F0PIuH5sEJLFMZR+wI72dzz1x1CSWD9pzwEP6xBoSohXlNw2wuWPFuAXHmMGzEldp4+BBq02uKiatKGV0caKzI4ss+/ufk8JrwIOpU7ZaWahgt/z8ROB0JhstqbOjTyZay9Y2wPTlAi3lSHzO5+RR1NP6+ACTiwYq/zu6ynNNjp+feDpNRXJ3aqVWFOxC9l2s8ePeKVhXr6QElwr/3vcxh9pRs0K+b+4TnoiTzscXD3M7ztlpwr1ZXguUJ7Jh2ORDMHhU1LHZU52ZPtRISIg54iVrF4Udal4/hGhLMYZQ2fZTo4Y1Z631qf07JSROoESGkZF7+gjuBjxnWtexYw83858OgimxKZVHW3BbcLzFJXGcpG7wCNxRLDWTPPwj0yQ8v7PgaU/v5ueowkhnqVsJMjSDYicoPvb2VJE6w71pjqOC0uuHoIk8EpekS1iLdhBnSFe5GeYuxI7AphMO4GTZJpE+IoaF/K2V6jBjrxKwVk= ceph-ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:34:05.821 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T21:34:06.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:05 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2896384561' entity='client.admin' 2024-09-13T21:34:09.265 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T21:34:09.266 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T21:34:09.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:09 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1074755747' entity='client.admin' 2024-09-13T21:34:11.308 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:11 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:11.308 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:11 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi154", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:34:11.308 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:11 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:11.308 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:11 smithi154 ceph-mon[23700]: Deploying daemon alertmanager.smithi154 on smithi154 2024-09-13T21:34:12.480 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi181 2024-09-13T21:34:12.481 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-13T21:34:12.481 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.conf 2024-09-13T21:34:12.500 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-13T21:34:12.500 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:34:12.558 INFO:tasks.cephadm:Adding host smithi181 to orchestrator... 2024-09-13T21:34:12.559 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch host add smithi181 2024-09-13T21:34:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:12 smithi154 ceph-mon[23700]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:34:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:12 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:16.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:16 smithi154 ceph-mon[23700]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:34:16.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:16 smithi154 ceph-mon[23700]: Deploying cephadm binary to smithi181 2024-09-13T21:34:16.626 INFO:teuthology.orchestra.run.smithi154.stdout:Added host 'smithi181' with addr '172.21.15.181' 2024-09-13T21:34:17.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:17 smithi154 ceph-mon[23700]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:17.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:17 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:17.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:17 smithi154 ceph-mon[23700]: Added host smithi181 2024-09-13T21:34:18.151 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T21:34:19.907 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:19 smithi154 ceph-mon[23700]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:20.396 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:34:20.396 INFO:teuthology.orchestra.run.smithi154.stdout:[{"addr": "172.21.15.154", "hostname": "smithi154", "labels": [], "status": ""}, {"addr": "172.21.15.181", "hostname": "smithi181", "labels": [], "status": ""}] 2024-09-13T21:34:21.574 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T21:34:21.574 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T21:34:22.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:21 smithi154 ceph-mon[23700]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:34:22.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:21 smithi154 ceph-mon[23700]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:22.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:21 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:22.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:21 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:34:22.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:21 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T21:34:22.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:21 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:34:22.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:21 smithi154 ceph-mon[23700]: Deploying daemon crash.smithi154 on smithi154 2024-09-13T21:34:22.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:21 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:24.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:23 smithi154 ceph-mon[23700]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:24.971 INFO:teuthology.orchestra.run.smithi154.stderr:adjusted tunables profile to default 2024-09-13T21:34:25.274 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:24 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/810592244' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T21:34:25.274 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:24 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:25.274 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:24 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:25.274 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:24 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:25.275 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:24 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:34:25.275 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:24 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:25.562 INFO:tasks.cephadm:Adding mon.smithi154 on smithi154 2024-09-13T21:34:25.562 INFO:tasks.cephadm:Adding mon.smithi181 on smithi181 2024-09-13T21:34:25.562 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi154:172.21.15.154=smithi154;smithi181:172.21.15.181=smithi181' 2024-09-13T21:34:26.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:25 smithi154 ceph-mon[23700]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:26.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:25 smithi154 ceph-mon[23700]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:34:26.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:25 smithi154 ceph-mon[23700]: Deploying daemon grafana.smithi154 on smithi154 2024-09-13T21:34:26.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:25 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/810592244' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T21:34:26.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:25 smithi154 ceph-mon[23700]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:34:27.334 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mon update... 2024-09-13T21:34:27.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:27 smithi154 ceph-mon[23700]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:27.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:27 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:27.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:27 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:27.914 DEBUG:teuthology.orchestra.run.smithi181:mon.smithi181> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi181.service 2024-09-13T21:34:27.917 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:27.917 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:28.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:28 smithi154 ceph-mon[23700]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi154:172.21.15.154=smithi154;smithi181:172.21.15.181=smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:34:28.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:28 smithi154 ceph-mon[23700]: Saving service mon spec with placement smithi154:172.21.15.154=smithi154;smithi181:172.21.15.181=smithi181;count:2 2024-09-13T21:34:29.650 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:29.650 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:29.652 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:34:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:29 smithi154 ceph-mon[23700]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:30.833 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:30 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/3758647606' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:34:31.098 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:31.098 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:32.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:32 smithi154 ceph-mon[23700]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:32.890 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:32.890 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:32.892 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:34:33.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:33 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/3495901326' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:34:34.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:34 smithi154 ceph-mon[23700]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:34.476 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:34.476 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:36.115 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:36 smithi154 ceph-mon[23700]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:36.115 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:36 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:36.135 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:36.135 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:36.137 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:34:37.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:37 smithi154 ceph-mon[23700]: Deploying daemon node-exporter.smithi154 on smithi154 2024-09-13T21:34:37.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:37 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/1191918665' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:34:37.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:37 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:37.730 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:37.730 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:38.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:38 smithi154 ceph-mon[23700]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:39.607 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:39.607 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:39.609 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:34:40.316 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:39 smithi154 ceph-mon[23700]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:34:40.316 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:39 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:40.316 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:39 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T21:34:40.316 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:39 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/3410496217' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:34:41.067 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:40 smithi154 ceph-mon[23700]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T21:34:41.067 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:40 smithi154 ceph-mon[23700]: mgrmap e14: smithi154.ayrwdi(active, since 43s) 2024-09-13T21:34:41.200 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:41.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:43.089 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:43.089 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:43.091 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:34:43.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:43 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/2933920250' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:34:44.652 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:44.654 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:46.557 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:46.557 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:46.560 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: Active manager daemon smithi154.ayrwdi restarted 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: Activating manager daemon smithi154.ayrwdi 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: mgrmap e15: smithi154.ayrwdi(active, starting, since 0.0364333s) 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:34:47.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/1992523189' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: [13/Sep/2024:21:34:46] ENGINE Bus STARTING 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: [13/Sep/2024:21:34:46] ENGINE Serving on https://172.21.15.154:7150 2024-09-13T21:34:47.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: [13/Sep/2024:21:34:46] ENGINE Bus STARTED 2024-09-13T21:34:47.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:48.246 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:48.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:48.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:48 smithi154 ceph-mon[23700]: mgrmap e16: smithi154.ayrwdi(active, since 1.03918s) 2024-09-13T21:34:48.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:48 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:49.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:49 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:49.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:49 smithi154 ceph-mon[23700]: mgrmap e17: smithi154.ayrwdi(active, since 2s) 2024-09-13T21:34:51.458 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:51.459 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:51.460 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:34:51.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:51 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/4227229762' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:34:51.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:51 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:53.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:53.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi154", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:34:53.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:34:53.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:52 smithi154 ceph-mon[23700]: Deploying daemon crash.smithi181 on smithi181 2024-09-13T21:34:53.721 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:53.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:55.731 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:55.731 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:55.732 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:34:56.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:56 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/2340552568' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:34:57.260 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:34:57.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:34:57.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:34:57.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:34:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T21:34:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:34:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:34:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:57 smithi154 ceph-mon[23700]: Deploying daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:34:59.671 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:34:59.671 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:34:59.672 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:35:00.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:34:59 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/2432725381' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:01.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:00 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:01.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:00 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:35:01.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:00 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:01.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:00 smithi154 ceph-mon[23700]: Deploying daemon mon.smithi181 on smithi181 2024-09-13T21:35:01.500 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:35:01.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:35:02.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:02 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:04.789 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:35:04.789 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:32:51.022089Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T21:35:04.790 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-09-13T21:35:04.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 podman[25754]: 2024-09-13 21:35:04.88035242 +0000 UTC m=+0.603597719 container start 690acc2b7d671bde24b1f4dcf40fc9f9d1c9f49bde35dbe3bf86dc9ad57583f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, version=8, release=754, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T21:35:05.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:04 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/2825831421' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.249 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T21:35:05.249 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T21:35:05.249 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pidfile_write: ignore empty --pid-file 2024-09-13T21:35:05.249 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: load: jerasure load: lrc 2024-09-13T21:35:05.249 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: RocksDB version: 6.15.5 2024-09-13T21:35:05.249 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T21:35:05.249 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Compile date Apr 18 2022 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: DB SUMMARY 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: DB Session ID: N9474WBEMJ0G3R5Y7G2P 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: CURRENT file: CURRENT 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi181/store.db dir, Total Num: 0, files: 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi181/store.db: 000004.log size: 511 ; 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.error_if_exists: 0 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.create_if_missing: 0 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.paranoid_checks: 1 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.env: 0x55bafc6c6860 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.fs: Posix File System 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.info_log: 0x55bafd1affa0 2024-09-13T21:35:05.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.statistics: (nil) 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.use_fsync: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_log_file_size: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.allow_fallocate: 1 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.use_direct_reads: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T21:35:05.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.db_log_dir: 2024-09-13T21:35:05.252 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi181/store.db 2024-09-13T21:35:05.252 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T21:35:05.252 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T21:35:05.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.write_buffer_manager: 0x55bafd2a02d0 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T21:35:05.254 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.unordered_write: 0 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.row_cache: None 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.wal_filter: None 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.preserve_deletes: 0 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.two_write_queues: 0 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T21:35:05.255 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.atomic_flush: 0 2024-09-13T21:35:05.256 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T21:35:05.256 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T21:35:05.256 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T21:35:05.256 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.log_readahead_size: 0 2024-09-13T21:35:05.256 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_background_jobs: 2 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_background_compactions: -1 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_subcompactions: 1 2024-09-13T21:35:05.257 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_open_files: -1 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_background_flushes: -1 2024-09-13T21:35:05.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Compression algorithms supported: 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: kZSTD supported: 0 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: kXpressCompression supported: 0 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: kLZ4Compression supported: 1 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: kBZip2Compression supported: 0 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: kZlibCompression supported: 1 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: kSnappyCompression supported: 1 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi181/store.db/MANIFEST-000003 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T21:35:05.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.merge_operator: 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_filter: None 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bafd17bd00) 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: cache_index_and_filter_blocks: 1 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: pin_top_level_index_and_filter: 1 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_type: 0 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_block_index_type: 0 2024-09-13T21:35:05.260 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_shortening: 1 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: hash_index_allow_collision: 1 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: checksum: 1 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: no_block_cache: 0 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache: 0x55bafd1e6170 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_name: BinnedLRUCache 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_options: 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: capacity : 536870912 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: num_shard_bits : 4 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: strict_capacity_limit : 0 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: high_pri_pool_ratio: 0.000 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_compressed: (nil) 2024-09-13T21:35:05.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: persistent_cache: (nil) 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_size: 4096 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_size_deviation: 10 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_restart_interval: 16 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_block_restart_interval: 1 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: metadata_block_size: 4096 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: partition_filters: 0 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: use_delta_encoding: 1 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: whole_key_filtering: 1 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: verify_compression: 0 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: read_amp_bytes_per_bit: 0 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: format_version: 4 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: enable_index_compression: 1 2024-09-13T21:35:05.262 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_align: 0 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compression: NoCompression 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.num_levels: 7 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T21:35:05.263 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T21:35:05.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T21:35:05.265 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T21:35:05.266 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.table_properties_collectors: 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.inplace_update_support: 0 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.bloom_locality: 0 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.max_successive_merges: 0 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T21:35:05.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.ttl: 2592000 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.enable_blob_files: false 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.min_blob_size: 0 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi181/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263304906577, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T21:35:05.268 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263304906885, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726263304, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "0894210b-79e2-4480-b9d6-1ef7b7a1086f", "db_session_id": "N9474WBEMJ0G3R5Y7G2P"}} 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263304907188, "job": 1, "event": "recovery_finished"} 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi181/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55bafd1cca80 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: DB pointer 0x55bafd1dc000 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** DB Stats ** 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T21:35:05.269 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-09-13T21:35:05.270 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:35:05.271 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T21:35:05.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181 does not exist in monmap, will attempt to join an existing cluster 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: using public_addr v2:172.21.15.181:0/0 -> [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: starting mon.smithi181 rank -1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi181 fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:35:05.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(???) e0 preinit fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).mds e1 new map 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).mds e1 print_map 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: e1 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: legacy client fscid: -1 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: No filesystems configured 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T21:35:05.274 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/810592244' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Deploying daemon grafana.smithi154 on smithi154 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/810592244' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T21:35:05.275 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi154:172.21.15.154=smithi154;smithi181:172.21.15.181=smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Saving service mon spec with placement smithi154:172.21.15.154=smithi154;smithi181:172.21.15.181=smithi181;count:2 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/3758647606' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/3495901326' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.276 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Deploying daemon node-exporter.smithi154 on smithi154 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/1191918665' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/3410496217' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14160 172.21.15.154:0/1044606394' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mgrmap e14: smithi154.ayrwdi(active, since 43s) 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/2933920250' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Active manager daemon smithi154.ayrwdi restarted 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Activating manager daemon smithi154.ayrwdi 2024-09-13T21:35:05.277 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mgrmap e15: smithi154.ayrwdi(active, starting, since 0.0364333s) 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:35:05.278 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/1992523189' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: [13/Sep/2024:21:34:46] ENGINE Bus STARTING 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: [13/Sep/2024:21:34:46] ENGINE Serving on https://172.21.15.154:7150 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: [13/Sep/2024:21:34:46] ENGINE Bus STARTED 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mgrmap e16: smithi154.ayrwdi(active, since 1.03918s) 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mgrmap e17: smithi154.ayrwdi(active, since 2s) 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/4227229762' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi154", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:35:05.279 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Deploying daemon crash.smithi181 on smithi181 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/2340552568' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Deploying daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/2432725381' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: Deploying daemon mon.smithi181 on smithi181 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/2825831421' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-13T21:35:05.281 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 ceph-mon[25811]: mon.smithi181@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-13T21:35:05.282 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 bash[25754]: 690acc2b7d671bde24b1f4dcf40fc9f9d1c9f49bde35dbe3bf86dc9ad57583f9 2024-09-13T21:35:05.282 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:04 smithi181 systemd[1]: Started Ceph mon.smithi181 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:35:06.690 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T21:35:06.690 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T21:35:10.245 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:35:10.245 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: mon.smithi154 calling monitor election 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/3842357559' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: mon.smithi181 calling monitor election 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: mon.smithi154 is new leader, mons smithi154,smithi181 in quorum (ranks 0,1) 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: monmap e2: 2 mons at {smithi154=[v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0],smithi181=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} 2024-09-13T21:35:10.246 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: fsmap 2024-09-13T21:35:10.247 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T21:35:10.247 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: mgrmap e17: smithi154.ayrwdi(active, since 23s) 2024-09-13T21:35:10.247 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:35:10.247 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/3842357559' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:35:10.247 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/3842357559' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:35:10.247 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: overall HEALTH_OK 2024-09-13T21:35:10.247 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/3842357559' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:35:10.247 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: mon.smithi154 calling monitor election 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/3842357559' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: mon.smithi181 calling monitor election 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: mon.smithi154 is new leader, mons smithi154,smithi181 in quorum (ranks 0,1) 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: monmap e2: 2 mons at {smithi154=[v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0],smithi181=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: fsmap 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: mgrmap e17: smithi154.ayrwdi(active, since 23s) 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/3842357559' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/3842357559' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: overall HEALTH_OK 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/3842357559' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:35:10.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:11.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:11 smithi154 ceph-mon[23700]: Deploying daemon node-exporter.smithi181 on smithi181 2024-09-13T21:35:11.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:11 smithi154 ceph-mon[23700]: mgrmap e18: smithi154.ayrwdi(active, since 23s), standbys: smithi181.byvkib 2024-09-13T21:35:11.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:11 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi181.byvkib", "id": "smithi181.byvkib"}]: dispatch 2024-09-13T21:35:11.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:11 smithi154 ceph-mon[23700]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:11.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:11 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:11 smithi181 ceph-mon[25811]: Deploying daemon node-exporter.smithi181 on smithi181 2024-09-13T21:35:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:11 smithi181 ceph-mon[25811]: mgrmap e18: smithi154.ayrwdi(active, since 23s), standbys: smithi181.byvkib 2024-09-13T21:35:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:11 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi181.byvkib", "id": "smithi181.byvkib"}]: dispatch 2024-09-13T21:35:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:11 smithi181 ceph-mon[25811]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:11.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:11 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:35:11.586 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:35:11.587 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":2,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","modified":"2024-09-13T21:35:04.934281Z","created":"2024-09-13T21:32:51.022089Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi181","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T21:35:11.588 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 2 2024-09-13T21:35:12.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:12 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:12.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:12 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/1768452479' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:12.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:12 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:12.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:12 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/1768452479' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T21:35:13.106 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T21:35:13.107 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T21:35:13.390 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:13 smithi181 ceph-mon[25811]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:13.396 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:13 smithi154 ceph-mon[23700]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:14.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:14 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:14.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:14 smithi154 ceph-mon[23700]: Deploying daemon prometheus.smithi154 on smithi154 2024-09-13T21:35:14.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:14 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:14.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:14 smithi181 ceph-mon[25811]: Deploying daemon prometheus.smithi154 on smithi154 2024-09-13T21:35:15.093 INFO:teuthology.orchestra.run.smithi154.stdout:# minimal ceph.conf for ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:35:15.093 INFO:teuthology.orchestra.run.smithi154.stdout:[global] 2024-09-13T21:35:15.093 INFO:teuthology.orchestra.run.smithi154.stdout: fsid = ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:35:15.093 INFO:teuthology.orchestra.run.smithi154.stdout: mon_host = [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] 2024-09-13T21:35:15.809 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T21:35:15.810 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:35:15.810 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T21:35:15.837 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:35:15.838 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:35:15.902 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-13T21:35:15.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T21:35:15.936 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-13T21:35:15.936 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:35:16.004 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T21:35:16.005 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:35:16.005 DEBUG:teuthology.orchestra.run.smithi154:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T21:35:16.021 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T21:35:16.021 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_1 2024-09-13T21:35:16.076 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T21:35:16.077 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:35:16.077 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-13T21:35:16.077 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:35:16.077 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:35:16.077 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-09-13 21:34:36.628788819 +0000 2024-09-13T21:35:16.077 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-09-13 21:32:07.831472538 +0000 2024-09-13T21:35:16.077 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-09-13 21:32:07.831472538 +0000 2024-09-13T21:35:16.077 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-09-13 21:32:07.831472538 +0000 2024-09-13T21:35:16.077 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T21:35:16.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:15 smithi154 ceph-mon[23700]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:16.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:15 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3936949287' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:16.112 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-09-13T21:35:16.112 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-09-13T21:35:16.112 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000201591 s, 2.5 MB/s 2024-09-13T21:35:16.113 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T21:35:16.170 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_2 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-09-13 21:34:36.628788819 +0000 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-09-13 21:32:08.163463036 +0000 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-09-13 21:32:08.163463036 +0000 2024-09-13T21:35:16.226 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-09-13 21:32:08.163463036 +0000 2024-09-13T21:35:16.227 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T21:35:16.294 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-09-13T21:35:16.294 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-09-13T21:35:16.294 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000202063 s, 2.5 MB/s 2024-09-13T21:35:16.296 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T21:35:16.354 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_3 2024-09-13T21:35:16.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:15 smithi181 ceph-mon[25811]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:16.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:15 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/3936949287' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:16.411 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T21:35:16.412 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:35:16.412 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-13T21:35:16.412 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:35:16.412 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:35:16.412 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-09-13 21:34:36.628788819 +0000 2024-09-13T21:35:16.412 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-09-13 21:32:08.487453764 +0000 2024-09-13T21:35:16.412 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-09-13 21:32:08.487453764 +0000 2024-09-13T21:35:16.412 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-09-13 21:32:08.487453764 +0000 2024-09-13T21:35:16.412 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T21:35:16.481 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-09-13T21:35:16.481 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-09-13T21:35:16.481 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000179421 s, 2.9 MB/s 2024-09-13T21:35:16.482 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T21:35:16.540 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_4 2024-09-13T21:35:16.599 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T21:35:16.599 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:35:16.599 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-13T21:35:16.599 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:35:16.599 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:35:16.599 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-09-13 21:34:36.629788791 +0000 2024-09-13T21:35:16.599 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-09-13 21:32:08.816444348 +0000 2024-09-13T21:35:16.600 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-09-13 21:32:08.816444348 +0000 2024-09-13T21:35:16.600 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-09-13 21:32:08.816444348 +0000 2024-09-13T21:35:16.600 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T21:35:16.663 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-09-13T21:35:16.663 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-09-13T21:35:16.663 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000197053 s, 2.6 MB/s 2024-09-13T21:35:16.664 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T21:35:16.721 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-09-13T21:35:16.722 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T21:35:16.741 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T21:35:16.741 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_1 2024-09-13T21:35:16.797 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T21:35:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:35:16.798 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-13T21:35:16.798 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:35:16.798 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:35:16.798 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-09-13 21:35:10.531957499 +0000 2024-09-13T21:35:16.798 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-09-13 21:32:07.834468846 +0000 2024-09-13T21:35:16.798 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-09-13 21:32:07.834468846 +0000 2024-09-13T21:35:16.798 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-09-13 21:32:07.834468846 +0000 2024-09-13T21:35:16.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T21:35:16.861 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-09-13T21:35:16.861 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-09-13T21:35:16.861 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.00025935 s, 2.0 MB/s 2024-09-13T21:35:16.863 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T21:35:16.920 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_2 2024-09-13T21:35:16.976 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T21:35:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:35:16.977 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-13T21:35:16.977 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:35:16.977 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:35:16.977 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-09-13 21:35:10.531957499 +0000 2024-09-13T21:35:16.977 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-09-13 21:32:08.173462247 +0000 2024-09-13T21:35:16.977 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-09-13 21:32:08.173462247 +0000 2024-09-13T21:35:16.977 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-09-13 21:32:08.173462247 +0000 2024-09-13T21:35:16.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T21:35:17.040 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-09-13T21:35:17.040 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-09-13T21:35:17.040 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000192907 s, 2.7 MB/s 2024-09-13T21:35:17.041 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T21:35:17.098 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_3 2024-09-13T21:35:17.154 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T21:35:17.154 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:35:17.154 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-13T21:35:17.154 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:35:17.155 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:35:17.155 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-09-13 21:35:10.532957479 +0000 2024-09-13T21:35:17.155 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-09-13 21:32:08.495455980 +0000 2024-09-13T21:35:17.155 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-09-13 21:32:08.495455980 +0000 2024-09-13T21:35:17.155 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-09-13 21:32:08.495455980 +0000 2024-09-13T21:35:17.155 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T21:35:17.218 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-09-13T21:35:17.218 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-09-13T21:35:17.218 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000223089 s, 2.3 MB/s 2024-09-13T21:35:17.219 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T21:35:17.276 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_4 2024-09-13T21:35:17.332 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T21:35:17.332 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T21:35:17.332 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T21:35:17.332 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T21:35:17.332 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T21:35:17.333 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-09-13 21:35:10.532957479 +0000 2024-09-13T21:35:17.333 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-09-13 21:32:08.830449459 +0000 2024-09-13T21:35:17.333 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-09-13 21:32:08.830449459 +0000 2024-09-13T21:35:17.333 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-09-13 21:32:08.830449459 +0000 2024-09-13T21:35:17.333 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T21:35:17.393 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:17 smithi181 ceph-mon[25811]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:17.393 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:17 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:17.396 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-09-13T21:35:17.396 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-09-13T21:35:17.396 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000200827 s, 2.5 MB/s 2024-09-13T21:35:17.397 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T21:35:17.453 INFO:tasks.cephadm:Deploying osd.0 on smithi154 with /dev/vg_nvme/lv_4... 2024-09-13T21:35:17.454 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T21:35:17.478 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:17 smithi154 ceph-mon[23700]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:17.478 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:17 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:19.316 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:19 smithi154 ceph-mon[23700]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:19.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:19 smithi181 ceph-mon[25811]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:21.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:21 smithi154 ceph-mon[23700]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:21 smithi181 ceph-mon[25811]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:23.391 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:23 smithi154 ceph-mon[23700]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:23 smithi181 ceph-mon[25811]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:25.354 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:25 smithi154 ceph-mon[23700]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:25 smithi181 ceph-mon[25811]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:26.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:26.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:35:26.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:26.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:35:26.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:27.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:27.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:35:27.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:27.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:35:27.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:27.202 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:35:27.220 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi154:vg_nvme/lv_4 2024-09-13T21:35:27.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:27 smithi154 ceph-mon[23700]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:27.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:27 smithi181 ceph-mon[25811]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:29.000 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:28 smithi154 ceph-mon[23700]: Updating smithi181:/etc/ceph/ceph.conf 2024-09-13T21:35:29.000 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:28 smithi154 ceph-mon[23700]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:35:29.001 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:28 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:28 smithi181 ceph-mon[25811]: Updating smithi181:/etc/ceph/ceph.conf 2024-09-13T21:35:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:28 smithi181 ceph-mon[25811]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:35:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:28 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:30.010 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:29 smithi154 ceph-mon[23700]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:30.010 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:29 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:35:30.010 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:29 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:35:30.010 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:29 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:29 smithi181 ceph-mon[25811]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:29 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:35:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:29 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:35:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:29 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:30.827 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:30 smithi154 ceph-mon[23700]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:35:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:30 smithi181 ceph-mon[25811]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:35:32.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:31 smithi154 ceph-mon[23700]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:32.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:31 smithi181 ceph-mon[25811]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:33.050 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:32 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2306563837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fde9e1d4-1248-4c90-8ec2-9266a3d387b8"}]: dispatch 2024-09-13T21:35:33.050 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:32 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2306563837' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fde9e1d4-1248-4c90-8ec2-9266a3d387b8"}]': finished 2024-09-13T21:35:33.051 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:32 smithi154 ceph-mon[23700]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T21:35:33.051 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:32 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:33.051 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:32 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3657611526' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:35:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:32 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2306563837' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fde9e1d4-1248-4c90-8ec2-9266a3d387b8"}]: dispatch 2024-09-13T21:35:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:32 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2306563837' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fde9e1d4-1248-4c90-8ec2-9266a3d387b8"}]': finished 2024-09-13T21:35:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:32 smithi181 ceph-mon[25811]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T21:35:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:32 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:32 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/3657611526' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:35:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:33 smithi154 ceph-mon[23700]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:34.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:34.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:35:34.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:35:34.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:33 smithi181 ceph-mon[25811]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:35:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:35:34.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:34.939 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:34 smithi154 ceph-mon[23700]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:35:34.939 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:34 smithi154 ceph-mon[23700]: Reconfiguring mon.smithi154 (unknown last config time)... 2024-09-13T21:35:34.939 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:34 smithi154 ceph-mon[23700]: Reconfiguring daemon mon.smithi154 on smithi154 2024-09-13T21:35:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:34 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:35:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:34 smithi181 ceph-mon[25811]: Reconfiguring mon.smithi154 (unknown last config time)... 2024-09-13T21:35:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:34 smithi181 ceph-mon[25811]: Reconfiguring daemon mon.smithi154 on smithi154 2024-09-13T21:35:36.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:35 smithi181 ceph-mon[25811]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:36.079 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:35 smithi154 ceph-mon[23700]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:37.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:37 smithi181 ceph-mon[25811]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:37.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:37 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:37.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:37 smithi181 ceph-mon[25811]: Reconfiguring mgr.smithi154.ayrwdi (unknown last config time)... 2024-09-13T21:35:37.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:37 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.ayrwdi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:35:37.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:37 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:35:37.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:37 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:37.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:37 smithi181 ceph-mon[25811]: Reconfiguring daemon mgr.smithi154.ayrwdi on smithi154 2024-09-13T21:35:37.738 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:37 smithi154 ceph-mon[23700]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:37.738 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:37 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:37.738 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:37 smithi154 ceph-mon[23700]: Reconfiguring mgr.smithi154.ayrwdi (unknown last config time)... 2024-09-13T21:35:37.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:37 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.ayrwdi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:35:37.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:37 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:35:37.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:37 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:37.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:37 smithi154 ceph-mon[23700]: Reconfiguring daemon mgr.smithi154.ayrwdi on smithi154 2024-09-13T21:35:39.793 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:39 smithi154 ceph-mon[23700]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:39.793 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:39 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:39.793 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:39 smithi154 ceph-mon[23700]: Reconfiguring alertmanager.smithi154 (dependencies changed)... 2024-09-13T21:35:39.793 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:39 smithi154 ceph-mon[23700]: Reconfiguring daemon alertmanager.smithi154 on smithi154 2024-09-13T21:35:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:39 smithi181 ceph-mon[25811]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:39 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:39 smithi181 ceph-mon[25811]: Reconfiguring alertmanager.smithi154 (dependencies changed)... 2024-09-13T21:35:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:39 smithi181 ceph-mon[25811]: Reconfiguring daemon alertmanager.smithi154 on smithi154 2024-09-13T21:35:40.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:35:40.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:40.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:40 smithi154 ceph-mon[23700]: Deploying daemon osd.0 on smithi154 2024-09-13T21:35:40.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:35:40.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:40.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:40 smithi181 ceph-mon[25811]: Deploying daemon osd.0 on smithi154 2024-09-13T21:35:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:41 smithi154 ceph-mon[23700]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:41 smithi181 ceph-mon[25811]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:43.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:43 smithi154 ceph-mon[23700]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:43 smithi181 ceph-mon[25811]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:45.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:45 smithi154 ceph-mon[23700]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:45.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:45.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:45 smithi154 ceph-mon[23700]: Reconfiguring crash.smithi154 (monmap changed)... 2024-09-13T21:35:45.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:35:45.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:45.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:45 smithi154 ceph-mon[23700]: Reconfiguring daemon crash.smithi154 on smithi154 2024-09-13T21:35:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:45 smithi181 ceph-mon[25811]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:45 smithi181 ceph-mon[25811]: Reconfiguring crash.smithi154 (monmap changed)... 2024-09-13T21:35:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:35:45.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:35:45.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:45 smithi181 ceph-mon[25811]: Reconfiguring daemon crash.smithi154 on smithi154 2024-09-13T21:35:46.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:35:46.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:35:46.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:35:46.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:35:47.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:47 smithi154 ceph-mon[23700]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:47.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:47 smithi181 ceph-mon[25811]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:49.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:49 smithi154 ceph-mon[23700]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:49 smithi181 ceph-mon[25811]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:51.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:51 smithi154 ceph-mon[23700]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:51.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:51 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:51 smithi181 ceph-mon[25811]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:51 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:53.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:53 smithi154 ceph-mon[23700]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:53.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:53.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:53 smithi154 ceph-mon[23700]: Reconfiguring grafana.smithi154 (dependencies changed)... 2024-09-13T21:35:53.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:53 smithi154 ceph-mon[23700]: Reconfiguring daemon grafana.smithi154 on smithi154 2024-09-13T21:35:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:53 smithi181 ceph-mon[25811]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:53 smithi181 ceph-mon[25811]: Reconfiguring grafana.smithi154 (dependencies changed)... 2024-09-13T21:35:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:53 smithi181 ceph-mon[25811]: Reconfiguring daemon grafana.smithi154 on smithi154 2024-09-13T21:35:54.525 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 0 on host 'smithi154' 2024-09-13T21:35:54.822 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:54 smithi154 ceph-mon[23700]: from='osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:35:54.823 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:54 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:54.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:54 smithi181 ceph-mon[25811]: from='osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:35:54.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:54 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:35:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:55 smithi181 ceph-mon[25811]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:55 smithi181 ceph-mon[25811]: from='osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T21:35:55.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:55 smithi181 ceph-mon[25811]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T21:35:55.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:55 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:55.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:55 smithi181 ceph-mon[25811]: from='osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:35:55.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:55 smithi154 ceph-mon[23700]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:55.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:55 smithi154 ceph-mon[23700]: from='osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T21:35:55.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:55 smithi154 ceph-mon[23700]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T21:35:55.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:55 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:55.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:55 smithi154 ceph-mon[23700]: from='osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:35:56.772 DEBUG:teuthology.orchestra.run.smithi154:osd.0> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T21:35:56.775 INFO:tasks.cephadm:Deploying osd.1 on smithi154 with /dev/vg_nvme/lv_3... 2024-09-13T21:35:56.775 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T21:35:56.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:56 smithi181 ceph-mon[25811]: from='osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-09-13T21:35:56.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:56 smithi181 ceph-mon[25811]: osdmap e8: 1 total, 0 up, 1 in 2024-09-13T21:35:56.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:56 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:56.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:56 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:56.938 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:56 smithi154 ceph-mon[23700]: from='osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-09-13T21:35:56.938 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:56 smithi154 ceph-mon[23700]: osdmap e8: 1 total, 0 up, 1 in 2024-09-13T21:35:56.938 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:56 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:56.938 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:56 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:57.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:57 smithi154 ceph-mon[23700]: purged_snaps scrub starts 2024-09-13T21:35:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:57 smithi154 ceph-mon[23700]: purged_snaps scrub ok 2024-09-13T21:35:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:57 smithi154 ceph-mon[23700]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:57 smithi154 ceph-mon[23700]: osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527] boot 2024-09-13T21:35:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:57 smithi154 ceph-mon[23700]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T21:35:57.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:57 smithi181 ceph-mon[25811]: purged_snaps scrub starts 2024-09-13T21:35:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:57 smithi181 ceph-mon[25811]: purged_snaps scrub ok 2024-09-13T21:35:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:57 smithi181 ceph-mon[25811]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T21:35:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:57 smithi181 ceph-mon[25811]: osd.0 [v2:172.21.15.154:6802/3023533527,v1:172.21.15.154:6803/3023533527] boot 2024-09-13T21:35:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:57 smithi181 ceph-mon[25811]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T21:35:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:35:58.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:58 smithi181 ceph-mon[25811]: osdmap e10: 1 total, 1 up, 1 in 2024-09-13T21:35:59.007 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:58 smithi154 ceph-mon[23700]: osdmap e10: 1 total, 1 up, 1 in 2024-09-13T21:35:59.796 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:35:59 smithi154 ceph-mon[23700]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:35:59.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:35:59 smithi181 ceph-mon[25811]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:00.826 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:00 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:00.826 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:00 smithi154 ceph-mon[23700]: Reconfiguring crash.smithi181 (monmap changed)... 2024-09-13T21:36:00.826 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:00 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:36:00.826 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:00 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:00.826 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:00 smithi154 ceph-mon[23700]: Reconfiguring daemon crash.smithi181 on smithi181 2024-09-13T21:36:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:00 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:00 smithi181 ceph-mon[25811]: Reconfiguring crash.smithi181 (monmap changed)... 2024-09-13T21:36:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:00 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:36:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:00 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:00.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:00 smithi181 ceph-mon[25811]: Reconfiguring daemon crash.smithi181 on smithi181 2024-09-13T21:36:01.024 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:36:01.043 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi154:vg_nvme/lv_3 2024-09-13T21:36:02.238 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:01 smithi181 ceph-mon[25811]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:02.238 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:01 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:02.238 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:01 smithi181 ceph-mon[25811]: Reconfiguring mgr.smithi181.byvkib (monmap changed)... 2024-09-13T21:36:02.238 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:01 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:36:02.238 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:01 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:36:02.238 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:01 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:02.238 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:01 smithi181 ceph-mon[25811]: Reconfiguring daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:36:02.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:01 smithi154 ceph-mon[23700]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:02.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:01 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:02.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:01 smithi154 ceph-mon[23700]: Reconfiguring mgr.smithi181.byvkib (monmap changed)... 2024-09-13T21:36:02.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:01 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:36:02.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:01 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:36:02.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:01 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:02.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:01 smithi154 ceph-mon[23700]: Reconfiguring daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:36:03.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: Reconfiguring mon.smithi181 (monmap changed)... 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: Reconfiguring daemon mon.smithi181 on smithi181 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:36:03.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:03.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:03.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:03.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: Reconfiguring mon.smithi181 (monmap changed)... 2024-09-13T21:36:03.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:36:03.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:36:03.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:03.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: Reconfiguring daemon mon.smithi181 on smithi181 2024-09-13T21:36:03.492 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:36:03.492 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:36:03.492 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:04.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:04 smithi154 ceph-mon[23700]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:36:04.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:04 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:04.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:04 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:36:04.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:04 smithi154 ceph-mon[23700]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:36:04.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:04 smithi181 ceph-mon[25811]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:36:04.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:04.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:04 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:36:04.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:04 smithi181 ceph-mon[25811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:36:05.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:05 smithi154 ceph-mon[23700]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:05.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:05 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/790353476' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63926304-68de-4f03-8384-f04b29d10496"}]: dispatch 2024-09-13T21:36:05.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:05 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/790353476' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63926304-68de-4f03-8384-f04b29d10496"}]': finished 2024-09-13T21:36:05.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:05 smithi154 ceph-mon[23700]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T21:36:05.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:05 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:05.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:05 smithi181 ceph-mon[25811]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:05.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:05 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/790353476' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63926304-68de-4f03-8384-f04b29d10496"}]: dispatch 2024-09-13T21:36:05.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:05 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/790353476' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63926304-68de-4f03-8384-f04b29d10496"}]': finished 2024-09-13T21:36:05.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:05 smithi181 ceph-mon[25811]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T21:36:05.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:05 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:06.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:06 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1244778505' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:36:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:06 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1244778505' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:36:07.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:07 smithi154 ceph-mon[23700]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:07.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:07 smithi181 ceph-mon[25811]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:09.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:09 smithi154 ceph-mon[23700]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:09.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:09 smithi181 ceph-mon[25811]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:11.823 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:11 smithi154 ceph-mon[23700]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:11.823 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:11 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:36:11.823 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:11 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:11 smithi181 ceph-mon[25811]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:11 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:36:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:11 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:12.800 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:12 smithi154 ceph-mon[23700]: Deploying daemon osd.1 on smithi154 2024-09-13T21:36:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:12 smithi181 ceph-mon[25811]: Deploying daemon osd.1 on smithi154 2024-09-13T21:36:13.838 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:13 smithi154 ceph-mon[23700]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:13 smithi181 ceph-mon[25811]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:15.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:15 smithi154 ceph-mon[23700]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:15 smithi181 ceph-mon[25811]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:17.968 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:17 smithi154 ceph-mon[23700]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:17.968 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:17 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:18.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:17 smithi181 ceph-mon[25811]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:18.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:17 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:19.440 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 1 on host 'smithi154' 2024-09-13T21:36:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.154:9093"}]: dispatch 2024-09-13T21:36:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:36:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.154:3000"}]: dispatch 2024-09-13T21:36:19.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:19.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:36:19.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.154:9095"}]: dispatch 2024-09-13T21:36:19.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:19.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:36:19.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:19.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.154:9093"}]: dispatch 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.154:3000"}]: dispatch 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.154:9095"}]: dispatch 2024-09-13T21:36:20.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:20.086 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:36:20.086 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:20.086 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:36:20.109 DEBUG:teuthology.orchestra.run.smithi154:osd.1> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T21:36:20.112 INFO:tasks.cephadm:Deploying osd.2 on smithi154 with /dev/vg_nvme/lv_2... 2024-09-13T21:36:20.112 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T21:36:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:20 smithi154 ceph-mon[23700]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.154:9093"}]: dispatch 2024-09-13T21:36:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:20 smithi154 ceph-mon[23700]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:36:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:20 smithi154 ceph-mon[23700]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.154:3000"}]: dispatch 2024-09-13T21:36:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:20 smithi154 ceph-mon[23700]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:36:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:20 smithi154 ceph-mon[23700]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.154:9095"}]: dispatch 2024-09-13T21:36:20.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:20 smithi154 ceph-mon[23700]: from='osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:36:20.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:20 smithi181 ceph-mon[25811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.154:9093"}]: dispatch 2024-09-13T21:36:20.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:20 smithi181 ceph-mon[25811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:36:20.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:20 smithi181 ceph-mon[25811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.154:3000"}]: dispatch 2024-09-13T21:36:20.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:20 smithi181 ceph-mon[25811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:36:20.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:20 smithi181 ceph-mon[25811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.154:9095"}]: dispatch 2024-09-13T21:36:20.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:20 smithi181 ceph-mon[25811]: from='osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:36:21.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:21 smithi154 ceph-mon[23700]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:21.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:21 smithi154 ceph-mon[23700]: from='osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T21:36:21.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:21 smithi154 ceph-mon[23700]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T21:36:21.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:21.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:21 smithi154 ceph-mon[23700]: from='osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:36:21.963 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:21.963 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:36:21 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[36987]: 2024-09-13T21:36:21.672+0000 7fe559631700 -1 osd.1 0 waiting for initial osdmap 2024-09-13T21:36:21.963 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:36:21 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[36987]: 2024-09-13T21:36:21.674+0000 7fe552fc6700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:36:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:21 smithi181 ceph-mon[25811]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:21 smithi181 ceph-mon[25811]: from='osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T21:36:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:21 smithi181 ceph-mon[25811]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T21:36:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:21 smithi181 ceph-mon[25811]: from='osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:36:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:22.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:22 smithi154 ceph-mon[23700]: from='osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-09-13T21:36:22.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:22 smithi154 ceph-mon[23700]: osdmap e13: 2 total, 1 up, 2 in 2024-09-13T21:36:22.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:22 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:22.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:22 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:22.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:22 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:22 smithi181 ceph-mon[25811]: from='osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-09-13T21:36:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:22 smithi181 ceph-mon[25811]: osdmap e13: 2 total, 1 up, 2 in 2024-09-13T21:36:23.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:22 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:23.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:22 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:23.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:22 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:23.950 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:23 smithi154 ceph-mon[23700]: purged_snaps scrub starts 2024-09-13T21:36:23.951 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:23 smithi154 ceph-mon[23700]: purged_snaps scrub ok 2024-09-13T21:36:23.951 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:23 smithi154 ceph-mon[23700]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:23.951 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:23 smithi154 ceph-mon[23700]: osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724] boot 2024-09-13T21:36:23.951 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:23 smithi154 ceph-mon[23700]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T21:36:23.951 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:23.955 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:36:23.976 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi154:vg_nvme/lv_2 2024-09-13T21:36:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:23 smithi181 ceph-mon[25811]: purged_snaps scrub starts 2024-09-13T21:36:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:23 smithi181 ceph-mon[25811]: purged_snaps scrub ok 2024-09-13T21:36:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:23 smithi181 ceph-mon[25811]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T21:36:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:23 smithi181 ceph-mon[25811]: osd.1 [v2:172.21.15.154:6810/3277122724,v1:172.21.15.154:6811/3277122724] boot 2024-09-13T21:36:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:23 smithi181 ceph-mon[25811]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T21:36:24.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:36:25.907 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:25 smithi154 ceph-mon[23700]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:25.907 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:25 smithi154 ceph-mon[23700]: osdmap e15: 2 total, 2 up, 2 in 2024-09-13T21:36:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:25 smithi181 ceph-mon[25811]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:25 smithi181 ceph-mon[25811]: osdmap e15: 2 total, 2 up, 2 in 2024-09-13T21:36:27.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:36:27.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:36:27.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:36:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:36:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:27.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:27 smithi154 ceph-mon[23700]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:27.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:27 smithi154 ceph-mon[23700]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:36:27.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:27 smithi181 ceph-mon[25811]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:27 smithi181 ceph-mon[25811]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:36:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:30.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:29 smithi154 ceph-mon[23700]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:29 smithi181 ceph-mon[25811]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:31.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:31.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:31.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:31.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:31.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:36:31.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:31.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:31.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:36:33.066 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:32 smithi154 ceph-mon[23700]: Detected new or changed devices on smithi154 2024-09-13T21:36:33.066 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:32 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi154 to 6432M 2024-09-13T21:36:33.066 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:32 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2040317482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5061bfe-56e7-4d7d-9999-a97d7fb4b79d"}]: dispatch 2024-09-13T21:36:33.067 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:32 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2040317482' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a5061bfe-56e7-4d7d-9999-a97d7fb4b79d"}]': finished 2024-09-13T21:36:33.067 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:32 smithi154 ceph-mon[23700]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T21:36:33.067 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:32 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:33.067 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:32 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:32 smithi181 ceph-mon[25811]: Detected new or changed devices on smithi154 2024-09-13T21:36:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:32 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi154 to 6432M 2024-09-13T21:36:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:32 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2040317482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5061bfe-56e7-4d7d-9999-a97d7fb4b79d"}]: dispatch 2024-09-13T21:36:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:32 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2040317482' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a5061bfe-56e7-4d7d-9999-a97d7fb4b79d"}]': finished 2024-09-13T21:36:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:32 smithi181 ceph-mon[25811]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T21:36:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:32 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:32 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:33 smithi154 ceph-mon[23700]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:33 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/865215631' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:36:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:33 smithi181 ceph-mon[25811]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:33 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/865215631' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:36:35.979 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:35 smithi154 ceph-mon[23700]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:35.979 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:35 smithi181 ceph-mon[25811]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:38.059 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:37 smithi154 ceph-mon[23700]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:37 smithi181 ceph-mon[25811]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:40.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:39 smithi154 ceph-mon[23700]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:40.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:39 smithi154 ceph-mon[23700]: Detected new or changed devices on smithi154 2024-09-13T21:36:40.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:39 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:40.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:39 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:39 smithi181 ceph-mon[25811]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:39 smithi181 ceph-mon[25811]: Detected new or changed devices on smithi154 2024-09-13T21:36:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:39 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:39 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:41.294 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:41 smithi154 ceph-mon[23700]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:41.294 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:41 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:36:41.294 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:41 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:41.295 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:41 smithi154 ceph-mon[23700]: Deploying daemon osd.2 on smithi154 2024-09-13T21:36:41.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:41 smithi181 ceph-mon[25811]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:41.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:41 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:36:41.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:41 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:41.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:41 smithi181 ceph-mon[25811]: Deploying daemon osd.2 on smithi154 2024-09-13T21:36:43.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:43 smithi154 ceph-mon[23700]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:43.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:43 smithi181 ceph-mon[25811]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:45.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:45 smithi154 ceph-mon[23700]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:45.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:45 smithi181 ceph-mon[25811]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:46.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:46.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:36:46.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:46.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:36:46.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:36:46.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:36:46.889 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:46.889 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:36:46.889 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:46.889 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:36:46.890 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:36:46.890 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:36:47.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:47 smithi154 ceph-mon[23700]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:47.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:47.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:47 smithi181 ceph-mon[25811]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:47.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:47 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:49.367 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 2 on host 'smithi154' 2024-09-13T21:36:49.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:49 smithi154 ceph-mon[23700]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:49.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:49 smithi154 ceph-mon[23700]: from='osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:36:49.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:49 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:49 smithi181 ceph-mon[25811]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:49 smithi181 ceph-mon[25811]: from='osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:36:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:49 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:49.989 DEBUG:teuthology.orchestra.run.smithi154:osd.2> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T21:36:49.992 INFO:tasks.cephadm:Deploying osd.3 on smithi154 with /dev/vg_nvme/lv_1... 2024-09-13T21:36:49.992 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T21:36:50.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:50 smithi154 ceph-mon[23700]: from='osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T21:36:50.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:50 smithi154 ceph-mon[23700]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T21:36:50.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:50 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:50.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:50 smithi154 ceph-mon[23700]: from='osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:36:50.840 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:36:50 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[40861]: 2024-09-13T21:36:50.598+0000 7f469695c700 -1 osd.2 0 waiting for initial osdmap 2024-09-13T21:36:50.840 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:36:50 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[40861]: 2024-09-13T21:36:50.601+0000 7f468f2ef700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:36:50.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:50 smithi181 ceph-mon[25811]: from='osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T21:36:50.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:50 smithi181 ceph-mon[25811]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T21:36:50.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:50 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:50.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:50 smithi181 ceph-mon[25811]: from='osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:36:51.706 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:51 smithi154 ceph-mon[23700]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:51.706 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:51 smithi154 ceph-mon[23700]: from='osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-09-13T21:36:51.706 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:51 smithi154 ceph-mon[23700]: osdmap e18: 3 total, 2 up, 3 in 2024-09-13T21:36:51.707 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:51 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:51.707 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:51 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:51.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:51 smithi181 ceph-mon[25811]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:51.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:51 smithi181 ceph-mon[25811]: from='osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-09-13T21:36:51.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:51 smithi181 ceph-mon[25811]: osdmap e18: 3 total, 2 up, 3 in 2024-09-13T21:36:51.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:51 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:51.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:51 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:52.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:52 smithi181 ceph-mon[25811]: purged_snaps scrub starts 2024-09-13T21:36:52.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:52 smithi181 ceph-mon[25811]: purged_snaps scrub ok 2024-09-13T21:36:52.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:52 smithi181 ceph-mon[25811]: osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451] boot 2024-09-13T21:36:52.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:52 smithi181 ceph-mon[25811]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T21:36:52.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:52 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:52.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:52 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T21:36:53.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:52 smithi154 ceph-mon[23700]: purged_snaps scrub starts 2024-09-13T21:36:53.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:52 smithi154 ceph-mon[23700]: purged_snaps scrub ok 2024-09-13T21:36:53.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:52 smithi154 ceph-mon[23700]: osd.2 [v2:172.21.15.154:6818/1362376451,v1:172.21.15.154:6819/1362376451] boot 2024-09-13T21:36:53.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:52 smithi154 ceph-mon[23700]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T21:36:53.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:36:53.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T21:36:53.891 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:53 smithi154 ceph-mon[23700]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:53.891 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T21:36:53.891 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:53 smithi154 ceph-mon[23700]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T21:36:53.891 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T21:36:53.896 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:36:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:53 smithi181 ceph-mon[25811]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T21:36:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T21:36:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:53 smithi181 ceph-mon[25811]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T21:36:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T21:36:53.915 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi154:vg_nvme/lv_1 2024-09-13T21:36:54.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:54 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T21:36:54.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:54 smithi154 ceph-mon[23700]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T21:36:54.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:54 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:54.666 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42304]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:36:54.667 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42304]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:54.667 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42304]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:54.667 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42323]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:36:54.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:54 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T21:36:54.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:54 smithi181 ceph-mon[25811]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T21:36:54.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:54 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:54.939 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42323]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:54.939 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42323]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:55.190 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42342]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:36:55.190 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42342]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:55.190 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:36:54 smithi154 sudo[42342]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:55.444 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42350]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:36:55.444 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42350]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:55.444 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42350]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:55.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:55 smithi154 ceph-mon[23700]: pgmap v73: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:36:55.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:55 smithi154 ceph-mon[23700]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T21:36:55.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:55 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:55.705 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42365]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:36:55.705 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42365]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:55.706 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42365]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:55 smithi181 ceph-mon[25811]: pgmap v73: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:36:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:55 smithi181 ceph-mon[25811]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T21:36:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:55 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:55.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42394]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:36:55.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42394]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:55.963 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42372]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:36:55.963 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42372]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:55.963 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42372]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:56.212 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:55 smithi154 sudo[42394]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:56.213 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 sudo[42402]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:36:56.482 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 sudo[42402]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:56.482 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 sudo[42402]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:56.706 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 sudo[28127]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T21:36:56.707 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 sudo[28127]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:56.707 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 sudo[28127]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:56.707 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 sudo[28130]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T21:36:56.707 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 sudo[28130]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T21:36:56.707 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 sudo[28130]: pam_unix(sudo:session): session closed for user root 2024-09-13T21:36:56.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 ceph-mon[23700]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:36:56.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 ceph-mon[23700]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:36:56.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:36:56.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:36:56.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:36:56.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:56 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:36:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 ceph-mon[25811]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:36:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 ceph-mon[25811]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:36:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:36:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:36:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:36:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:56 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:36:58.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:57 smithi154 ceph-mon[23700]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:36:58.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:57 smithi154 ceph-mon[23700]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:36:58.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:57 smithi154 ceph-mon[23700]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:36:58.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:57 smithi154 ceph-mon[23700]: mgrmap e19: smithi154.ayrwdi(active, since 2m), standbys: smithi181.byvkib 2024-09-13T21:36:58.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:58.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:36:58.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:36:58.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:57 smithi181 ceph-mon[25811]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:36:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:57 smithi181 ceph-mon[25811]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T21:36:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:57 smithi181 ceph-mon[25811]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T21:36:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:57 smithi181 ceph-mon[25811]: mgrmap e19: smithi154.ayrwdi(active, since 2m), standbys: smithi181.byvkib 2024-09-13T21:36:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:58.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:36:58.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:36:58.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:59.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:36:59.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:59.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:59.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:59.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:59.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:59.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:59.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:36:59.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:59.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:58 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:36:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:36:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:36:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:36:59.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:36:59.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:36:59.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:58 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:36:59.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:59 smithi154 ceph-mon[23700]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:36:59.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:36:59 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi154 to 4288M 2024-09-13T21:37:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:59 smithi181 ceph-mon[25811]: pgmap v76: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:36:59 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi154 to 4288M 2024-09-13T21:37:01.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:01 smithi154 ceph-mon[23700]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:01.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:01 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1205673685' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb1e4228-7c42-47a3-a6ee-1f613dfcc829"}]: dispatch 2024-09-13T21:37:01.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:01 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1205673685' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb1e4228-7c42-47a3-a6ee-1f613dfcc829"}]': finished 2024-09-13T21:37:01.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:01 smithi154 ceph-mon[23700]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T21:37:01.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:01 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:01.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:01 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2647759383' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:37:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:01 smithi181 ceph-mon[25811]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:01 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1205673685' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb1e4228-7c42-47a3-a6ee-1f613dfcc829"}]: dispatch 2024-09-13T21:37:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:01 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1205673685' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb1e4228-7c42-47a3-a6ee-1f613dfcc829"}]': finished 2024-09-13T21:37:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:01 smithi181 ceph-mon[25811]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T21:37:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:01 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:01 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2647759383' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:37:04.051 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:03 smithi154 ceph-mon[23700]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:04.051 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:04.051 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:03 smithi181 ceph-mon[25811]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:05.935 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:05 smithi154 ceph-mon[23700]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:06.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:05 smithi181 ceph-mon[25811]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:07.835 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:07 smithi154 ceph-mon[23700]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:07.835 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:37:07.835 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:07 smithi181 ceph-mon[25811]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:37:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:09.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:08 smithi154 ceph-mon[23700]: Deploying daemon osd.3 on smithi154 2024-09-13T21:37:09.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:08 smithi181 ceph-mon[25811]: Deploying daemon osd.3 on smithi154 2024-09-13T21:37:09.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:09 smithi154 ceph-mon[23700]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:09 smithi181 ceph-mon[25811]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:11.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:11 smithi154 ceph-mon[23700]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:11 smithi181 ceph-mon[25811]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:13.482 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:13 smithi154 ceph-mon[23700]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:13.482 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:13 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:13.482 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:13 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:13.482 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:13 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:13.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:13 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:13 smithi181 ceph-mon[25811]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:13 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:13 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:13 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:13 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:15.330 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 3 on host 'smithi154' 2024-09-13T21:37:15.500 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:15 smithi154 ceph-mon[23700]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:15.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:15 smithi181 ceph-mon[25811]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:15.959 DEBUG:teuthology.orchestra.run.smithi154:osd.3> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T21:37:15.962 INFO:tasks.cephadm:Deploying osd.4 on smithi181 with /dev/vg_nvme/lv_4... 2024-09-13T21:37:15.962 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T21:37:16.207 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:16 smithi154 ceph-mon[23700]: from='osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:37:16.207 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:16 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:16.272 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:16 smithi181 ceph-mon[25811]: from='osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:37:16.273 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:16 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:17.226 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:17 smithi154 ceph-mon[23700]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:17.226 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:17 smithi154 ceph-mon[23700]: from='osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T21:37:17.226 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:17 smithi154 ceph-mon[23700]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T21:37:17.226 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:17 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:17.227 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:17 smithi154 ceph-mon[23700]: from='osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:37:17.227 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:37:17 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[44590]: 2024-09-13T21:37:17.180+0000 7f719a5cd700 -1 osd.3 0 waiting for initial osdmap 2024-09-13T21:37:17.227 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:37:17 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[44590]: 2024-09-13T21:37:17.186+0000 7f7195765700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:37:17.481 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:17 smithi181 ceph-mon[25811]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:17.481 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:17 smithi181 ceph-mon[25811]: from='osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T21:37:17.481 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:17 smithi181 ceph-mon[25811]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T21:37:17.481 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:17 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:17.481 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:17 smithi181 ceph-mon[25811]: from='osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:37:18.452 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:18 smithi181 ceph-mon[25811]: from='osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-09-13T21:37:18.452 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:18 smithi181 ceph-mon[25811]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T21:37:18.452 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:18 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:18.452 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:18 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:18.457 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:37:18.477 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_4 2024-09-13T21:37:18.564 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:18 smithi154 ceph-mon[23700]: from='osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-09-13T21:37:18.565 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:18 smithi154 ceph-mon[23700]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T21:37:18.565 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:18 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:18.565 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:18 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:19.318 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:19 smithi181 ceph-mon[25811]: purged_snaps scrub starts 2024-09-13T21:37:19.319 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:19 smithi181 ceph-mon[25811]: purged_snaps scrub ok 2024-09-13T21:37:19.319 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:19 smithi181 ceph-mon[25811]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:19.319 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:19.319 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:19 smithi181 ceph-mon[25811]: osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664] boot 2024-09-13T21:37:19.319 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:19 smithi181 ceph-mon[25811]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T21:37:19.319 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:19.319 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:19.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:19 smithi154 ceph-mon[23700]: purged_snaps scrub starts 2024-09-13T21:37:19.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:19 smithi154 ceph-mon[23700]: purged_snaps scrub ok 2024-09-13T21:37:19.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:19 smithi154 ceph-mon[23700]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T21:37:19.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:19.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:19 smithi154 ceph-mon[23700]: osd.3 [v2:172.21.15.154:6826/616538664,v1:172.21.15.154:6827/616538664] boot 2024-09-13T21:37:19.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:19 smithi154 ceph-mon[23700]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T21:37:19.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:37:19.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:20.534 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:20 smithi154 ceph-mon[23700]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T21:37:20.553 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:20 smithi181 ceph-mon[25811]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T21:37:21.460 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:21 smithi154 ceph-mon[23700]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:21.460 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:21 smithi154 ceph-mon[23700]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:37:21.460 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:37:21.460 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:37:21.460 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:21.556 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:21 smithi181 ceph-mon[25811]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:21.556 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:21 smithi181 ceph-mon[25811]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:37:21.557 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:37:21.557 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:37:21.557 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: Detected new or changed devices on smithi154 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi154 to 3216M 2024-09-13T21:37:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/3940008286' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b704e20c-943d-42c1-b868-8cd7d317536c"}]: dispatch 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b704e20c-943d-42c1-b868-8cd7d317536c"}]: dispatch 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b704e20c-943d-42c1-b868-8cd7d317536c"}]': finished 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T21:37:23.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:23.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:23 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/2488642870' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:37:23.501 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:23.501 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: Detected new or changed devices on smithi154 2024-09-13T21:37:23.501 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:23.501 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:23.501 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi154 to 3216M 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/3940008286' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b704e20c-943d-42c1-b868-8cd7d317536c"}]: dispatch 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b704e20c-943d-42c1-b868-8cd7d317536c"}]: dispatch 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b704e20c-943d-42c1-b868-8cd7d317536c"}]': finished 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T21:37:23.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:23.503 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:23 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/2488642870' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:37:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:25 smithi181 ceph-mon[25811]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:25.469 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:25 smithi154 ceph-mon[23700]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:27.494 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:27 smithi181 ceph-mon[25811]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:27.494 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:27.494 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:27 smithi154 ceph-mon[23700]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:29.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:29 smithi181 ceph-mon[25811]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:29.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:29 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:37:29.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:29 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:29.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:29 smithi181 ceph-mon[25811]: Deploying daemon osd.4 on smithi181 2024-09-13T21:37:29.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:29 smithi154 ceph-mon[23700]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:29.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:29 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:37:29.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:29 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:29.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:29 smithi154 ceph-mon[23700]: Deploying daemon osd.4 on smithi181 2024-09-13T21:37:31.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:31 smithi181 ceph-mon[25811]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:31.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:31 smithi154 ceph-mon[23700]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:33.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:33 smithi181 ceph-mon[25811]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:33.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:33.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:33.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:33.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:33.488 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:33 smithi154 ceph-mon[23700]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:33.488 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:33.488 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:33.488 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:33.488 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:35.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:35 smithi181 ceph-mon[25811]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:35.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:35 smithi154 ceph-mon[23700]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:36.428 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 4 on host 'smithi181' 2024-09-13T21:37:37.008 DEBUG:teuthology.orchestra.run.smithi181:osd.4> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T21:37:37.010 INFO:tasks.cephadm:Deploying osd.5 on smithi181 with /dev/vg_nvme/lv_3... 2024-09-13T21:37:37.011 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T21:37:37.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:37 smithi181 ceph-mon[25811]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:37.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:37 smithi181 ceph-mon[25811]: from='osd.4 [v2:172.21.15.181:6800/2957526050,v1:172.21.15.181:6801/2957526050]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:37:37.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:37 smithi181 ceph-mon[25811]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:37:37.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:37 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:37.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:37 smithi154 ceph-mon[23700]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:37.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:37 smithi154 ceph-mon[23700]: from='osd.4 [v2:172.21.15.181:6800/2957526050,v1:172.21.15.181:6801/2957526050]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:37:37.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:37 smithi154 ceph-mon[23700]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:37:37.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:37 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:38.335 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:38 smithi181 ceph-mon[25811]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T21:37:38.335 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:38 smithi181 ceph-mon[25811]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T21:37:38.335 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:38 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:38.335 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:38 smithi181 ceph-mon[25811]: from='osd.4 [v2:172.21.15.181:6800/2957526050,v1:172.21.15.181:6801/2957526050]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:37:38.335 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:38 smithi181 ceph-mon[25811]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:37:38.335 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:38 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:38.336 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:37:38 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[30051]: 2024-09-13T21:37:38.231+0000 7f7ecb57a700 -1 osd.4 0 waiting for initial osdmap 2024-09-13T21:37:38.336 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:37:38 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[30051]: 2024-09-13T21:37:38.235+0000 7f7ec7f15700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:37:38.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:38 smithi154 ceph-mon[23700]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T21:37:38.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:38 smithi154 ceph-mon[23700]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T21:37:38.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:38 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:38.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:38 smithi154 ceph-mon[23700]: from='osd.4 [v2:172.21.15.181:6800/2957526050,v1:172.21.15.181:6801/2957526050]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:37:38.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:38 smithi154 ceph-mon[23700]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:37:38.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:38 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:39.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:39 smithi154 ceph-mon[23700]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:39.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:39 smithi154 ceph-mon[23700]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-13T21:37:39.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:39 smithi154 ceph-mon[23700]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T21:37:39.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:39 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:39.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:39 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:39.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:39 smithi181 ceph-mon[25811]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T21:37:39.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:39 smithi181 ceph-mon[25811]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-13T21:37:39.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:39 smithi181 ceph-mon[25811]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T21:37:39.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:39 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:39.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:39 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:40 smithi181 ceph-mon[25811]: purged_snaps scrub starts 2024-09-13T21:37:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:40 smithi181 ceph-mon[25811]: purged_snaps scrub ok 2024-09-13T21:37:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:40 smithi181 ceph-mon[25811]: osd.4 [v2:172.21.15.181:6800/2957526050,v1:172.21.15.181:6801/2957526050] boot 2024-09-13T21:37:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:40 smithi181 ceph-mon[25811]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T21:37:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:40 smithi181 ceph-mon[25811]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T21:37:40.430 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:37:40.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_3 2024-09-13T21:37:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:40 smithi154 ceph-mon[23700]: purged_snaps scrub starts 2024-09-13T21:37:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:40 smithi154 ceph-mon[23700]: purged_snaps scrub ok 2024-09-13T21:37:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:40 smithi154 ceph-mon[23700]: osd.4 [v2:172.21.15.181:6800/2957526050,v1:172.21.15.181:6801/2957526050] boot 2024-09-13T21:37:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:40 smithi154 ceph-mon[23700]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T21:37:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:37:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:40 smithi154 ceph-mon[23700]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T21:37:41.289 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:41 smithi181 ceph-mon[25811]: pgmap v106: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:41.289 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:41 smithi181 ceph-mon[25811]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T21:37:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:41 smithi154 ceph-mon[23700]: pgmap v106: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:41 smithi154 ceph-mon[23700]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T21:37:43.252 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:43.252 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T21:37:43.252 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: Detected new or changed devices on smithi181 2024-09-13T21:37:43.252 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:43.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:43.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi181 to 15935M 2024-09-13T21:37:43.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:43.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:43.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:43.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:43.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:43.253 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:43.391 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: Detected new or changed devices on smithi181 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi181 to 15935M 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:43.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:44.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:44 smithi181 ceph-mon[25811]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T21:37:44.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:44 smithi181 ceph-mon[25811]: from='client.24161 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:37:44.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:44 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:37:44.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:44 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:37:44.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:44 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:44.408 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:44 smithi154 ceph-mon[23700]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T21:37:44.408 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:44 smithi154 ceph-mon[23700]: from='client.24161 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:37:44.408 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:44 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:37:44.408 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:44 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:37:44.408 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:44 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:45.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:45 smithi181 ceph-mon[25811]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:37:45.578 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:45 smithi154 ceph-mon[23700]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:37:46.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:46 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/1063684491' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1daa049b-e978-47bb-b333-fd41cc4c3885"}]: dispatch 2024-09-13T21:37:46.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:46 smithi181 ceph-mon[25811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1daa049b-e978-47bb-b333-fd41cc4c3885"}]: dispatch 2024-09-13T21:37:46.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:46 smithi181 ceph-mon[25811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1daa049b-e978-47bb-b333-fd41cc4c3885"}]': finished 2024-09-13T21:37:46.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:46 smithi181 ceph-mon[25811]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T21:37:46.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:37:46.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:46 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/1894385561' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:37:46.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:46 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/1063684491' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1daa049b-e978-47bb-b333-fd41cc4c3885"}]: dispatch 2024-09-13T21:37:46.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:46 smithi154 ceph-mon[23700]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1daa049b-e978-47bb-b333-fd41cc4c3885"}]: dispatch 2024-09-13T21:37:46.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:46 smithi154 ceph-mon[23700]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1daa049b-e978-47bb-b333-fd41cc4c3885"}]': finished 2024-09-13T21:37:46.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:46 smithi154 ceph-mon[23700]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T21:37:46.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:37:46.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:46 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/1894385561' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:37:47.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:47 smithi181 ceph-mon[25811]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-13T21:37:47.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:47 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:37:47.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:47 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:37:47.593 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:47 smithi154 ceph-mon[23700]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-13T21:37:47.594 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:37:47.594 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:37:49.393 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:49 smithi181 ceph-mon[25811]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T21:37:49.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:49 smithi154 ceph-mon[23700]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T21:37:51.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:51 smithi181 ceph-mon[25811]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:37:51.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:51 smithi154 ceph-mon[23700]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:37:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:52 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:37:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:52 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:52 smithi181 ceph-mon[25811]: Deploying daemon osd.5 on smithi181 2024-09-13T21:37:52.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:37:52.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:52.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:52 smithi154 ceph-mon[23700]: Deploying daemon osd.5 on smithi181 2024-09-13T21:37:53.220 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:53 smithi181 ceph-mon[25811]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-09-13T21:37:53.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:53 smithi154 ceph-mon[23700]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-09-13T21:37:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:55 smithi181 ceph-mon[25811]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:55.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:55 smithi154 ceph-mon[23700]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:57 smithi181 ceph-mon[25811]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:57 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:57.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:57 smithi154 ceph-mon[23700]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:57.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:37:57.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:37:57.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:37:57.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:57 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:37:59.489 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:37:59 smithi181 ceph-mon[25811]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:37:59.548 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 5 on host 'smithi181' 2024-09-13T21:37:59.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:37:59 smithi154 ceph-mon[23700]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:38:00.177 DEBUG:teuthology.orchestra.run.smithi181:osd.5> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T21:38:00.180 INFO:tasks.cephadm:Deploying osd.6 on smithi181 with /dev/vg_nvme/lv_2... 2024-09-13T21:38:00.180 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T21:38:00.466 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:00 smithi181 ceph-mon[25811]: from='osd.5 [v2:172.21.15.181:6808/1539599335,v1:172.21.15.181:6809/1539599335]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:38:00.466 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:00 smithi181 ceph-mon[25811]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:38:00.466 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:00 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:00.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:00 smithi154 ceph-mon[23700]: from='osd.5 [v2:172.21.15.181:6808/1539599335,v1:172.21.15.181:6809/1539599335]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:38:00.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:00 smithi154 ceph-mon[23700]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:38:00.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:00 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-mon[25811]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:38:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-mon[25811]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T21:38:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-mon[25811]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T21:38:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-mon[25811]: from='osd.5 [v2:172.21.15.181:6808/1539599335,v1:172.21.15.181:6809/1539599335]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-mon[25811]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:01.651 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[33013]: 2024-09-13T21:38:01.405+0000 7f21d28b1700 -1 osd.5 0 waiting for initial osdmap 2024-09-13T21:38:01.651 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:38:01 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[33013]: 2024-09-13T21:38:01.410+0000 7f21cf24c700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:38:01.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:01 smithi154 ceph-mon[23700]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:38:01.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:01 smithi154 ceph-mon[23700]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T21:38:01.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:01 smithi154 ceph-mon[23700]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T21:38:01.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:01 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:01.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:01 smithi154 ceph-mon[23700]: from='osd.5 [v2:172.21.15.181:6808/1539599335,v1:172.21.15.181:6809/1539599335]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:01.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:01 smithi154 ceph-mon[23700]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:01.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:01 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:02.711 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:02 smithi181 ceph-mon[25811]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-13T21:38:02.711 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:02 smithi181 ceph-mon[25811]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T21:38:02.711 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:02 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:02.711 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:02 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:02.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:02 smithi154 ceph-mon[23700]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-13T21:38:02.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:02 smithi154 ceph-mon[23700]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T21:38:02.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:02 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:02.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:02 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:03.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:03 smithi181 ceph-mon[25811]: purged_snaps scrub starts 2024-09-13T21:38:03.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:03 smithi181 ceph-mon[25811]: purged_snaps scrub ok 2024-09-13T21:38:03.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:03 smithi181 ceph-mon[25811]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:38:03.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:03.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:03 smithi181 ceph-mon[25811]: osd.5 [v2:172.21.15.181:6808/1539599335,v1:172.21.15.181:6809/1539599335] boot 2024-09-13T21:38:03.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:03 smithi181 ceph-mon[25811]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T21:38:03.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:03 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:03.792 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:38:03.811 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_2 2024-09-13T21:38:03.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:03 smithi154 ceph-mon[23700]: purged_snaps scrub starts 2024-09-13T21:38:03.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:03 smithi154 ceph-mon[23700]: purged_snaps scrub ok 2024-09-13T21:38:03.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:03 smithi154 ceph-mon[23700]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T21:38:03.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:03.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:03 smithi154 ceph-mon[23700]: osd.5 [v2:172.21.15.181:6808/1539599335,v1:172.21.15.181:6809/1539599335] boot 2024-09-13T21:38:03.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:03 smithi154 ceph-mon[23700]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T21:38:03.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:03 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:38:04.550 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:04 smithi181 ceph-mon[25811]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T21:38:04.550 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:04 smithi181 ceph-mon[25811]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T21:38:04.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:04 smithi154 ceph-mon[23700]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T21:38:04.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:04 smithi154 ceph-mon[23700]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T21:38:05.640 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:05 smithi181 ceph-mon[25811]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:05.641 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:05 smithi181 ceph-mon[25811]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:38:05.641 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:05 smithi181 ceph-mon[25811]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T21:38:05.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:05 smithi154 ceph-mon[23700]: pgmap v127: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:05.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:05 smithi154 ceph-mon[23700]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:38:05.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:05 smithi154 ceph-mon[23700]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T21:38:07.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: Detected new or changed devices on smithi181 2024-09-13T21:38:07.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:07.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:07.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi181 to 7967M 2024-09-13T21:38:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:06 smithi154 ceph-mon[23700]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T21:38:07.097 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: Detected new or changed devices on smithi181 2024-09-13T21:38:07.097 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi181 to 7967M 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:07.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:06 smithi181 ceph-mon[25811]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T21:38:08.027 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:07 smithi181 ceph-mon[25811]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:08.028 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:07 smithi181 ceph-mon[25811]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:38:08.028 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:38:08.028 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:38:08.028 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:08.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:07 smithi154 ceph-mon[23700]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:08.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:07 smithi154 ceph-mon[23700]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:38:08.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:38:08.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:38:08.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:10.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:09 smithi154 ceph-mon[23700]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:10.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:09 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/2769772821' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "549ac1b3-8f18-4b1b-b2b6-330115060298"}]: dispatch 2024-09-13T21:38:10.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:09 smithi154 ceph-mon[23700]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "549ac1b3-8f18-4b1b-b2b6-330115060298"}]: dispatch 2024-09-13T21:38:10.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:09 smithi154 ceph-mon[23700]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "549ac1b3-8f18-4b1b-b2b6-330115060298"}]': finished 2024-09-13T21:38:10.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:09 smithi154 ceph-mon[23700]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T21:38:10.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:09 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:10.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:09 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/2087454357' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:38:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:09 smithi181 ceph-mon[25811]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:09 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/2769772821' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "549ac1b3-8f18-4b1b-b2b6-330115060298"}]: dispatch 2024-09-13T21:38:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:09 smithi181 ceph-mon[25811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "549ac1b3-8f18-4b1b-b2b6-330115060298"}]: dispatch 2024-09-13T21:38:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:09 smithi181 ceph-mon[25811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "549ac1b3-8f18-4b1b-b2b6-330115060298"}]': finished 2024-09-13T21:38:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:09 smithi181 ceph-mon[25811]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T21:38:10.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:09 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:10.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:09 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/2087454357' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:38:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:11 smithi181 ceph-mon[25811]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-09-13T21:38:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:11 smithi181 ceph-mon[25811]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T21:38:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:11 smithi181 ceph-mon[25811]: Cluster is now healthy 2024-09-13T21:38:11.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:11 smithi154 ceph-mon[23700]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-09-13T21:38:11.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:11 smithi154 ceph-mon[23700]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T21:38:11.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:11 smithi154 ceph-mon[23700]: Cluster is now healthy 2024-09-13T21:38:13.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:13 smithi154 ceph-mon[23700]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T21:38:13.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:13 smithi181 ceph-mon[25811]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T21:38:15.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:15 smithi181 ceph-mon[25811]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:38:15.587 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:15 smithi154 ceph-mon[23700]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:38:16.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:16 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:38:16.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:16 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:16.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:16 smithi181 ceph-mon[25811]: Deploying daemon osd.6 on smithi181 2024-09-13T21:38:16.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:16 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:38:16.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:16 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:16.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:16 smithi154 ceph-mon[23700]: Deploying daemon osd.6 on smithi181 2024-09-13T21:38:17.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:17 smithi181 ceph-mon[25811]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:38:17.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:17 smithi154 ceph-mon[23700]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:38:19.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:19 smithi181 ceph-mon[25811]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:38:19.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:19 smithi154 ceph-mon[23700]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:38:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:21 smithi181 ceph-mon[25811]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:21.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:21 smithi154 ceph-mon[23700]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:21.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:21.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:21.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:21.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:23 smithi181 ceph-mon[25811]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:23.480 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 6 on host 'smithi181' 2024-09-13T21:38:23.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:23 smithi154 ceph-mon[23700]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:24.093 DEBUG:teuthology.orchestra.run.smithi181:osd.6> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T21:38:24.096 INFO:tasks.cephadm:Deploying osd.7 on smithi181 with /dev/vg_nvme/lv_1... 2024-09-13T21:38:24.096 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T21:38:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:24 smithi181 ceph-mon[25811]: from='osd.6 [v2:172.21.15.181:6816/1179366902,v1:172.21.15.181:6817/1179366902]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:38:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:24 smithi181 ceph-mon[25811]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:38:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:24 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:24.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:24 smithi154 ceph-mon[23700]: from='osd.6 [v2:172.21.15.181:6816/1179366902,v1:172.21.15.181:6817/1179366902]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:38:24.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:24 smithi154 ceph-mon[23700]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:38:24.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:24 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:25.185 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:25 smithi181 ceph-mon[25811]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:25.185 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:25 smithi181 ceph-mon[25811]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T21:38:25.185 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:25 smithi181 ceph-mon[25811]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T21:38:25.185 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:25 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:25.185 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:25 smithi181 ceph-mon[25811]: from='osd.6 [v2:172.21.15.181:6816/1179366902,v1:172.21.15.181:6817/1179366902]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:25.186 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:25 smithi181 ceph-mon[25811]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:25.186 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:38:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[35980]: 2024-09-13T21:38:25.158+0000 7f8554adf700 -1 osd.6 0 waiting for initial osdmap 2024-09-13T21:38:25.186 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:38:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[35980]: 2024-09-13T21:38:25.164+0000 7f854f476700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:38:25.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:25 smithi154 ceph-mon[23700]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:25.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:25 smithi154 ceph-mon[23700]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T21:38:25.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:25 smithi154 ceph-mon[23700]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T21:38:25.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:25 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:25.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:25 smithi154 ceph-mon[23700]: from='osd.6 [v2:172.21.15.181:6816/1179366902,v1:172.21.15.181:6817/1179366902]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:25.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:25 smithi154 ceph-mon[23700]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:26.550 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:26 smithi181 ceph-mon[25811]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-13T21:38:26.550 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:26 smithi181 ceph-mon[25811]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T21:38:26.550 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:26.550 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:26.551 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:26.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:26 smithi154 ceph-mon[23700]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-13T21:38:26.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:26 smithi154 ceph-mon[23700]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T21:38:26.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:26.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:26.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:26.814 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-09-13T21:38:26.831 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_1 2024-09-13T21:38:27.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:27 smithi181 ceph-mon[25811]: purged_snaps scrub starts 2024-09-13T21:38:27.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:27 smithi181 ceph-mon[25811]: purged_snaps scrub ok 2024-09-13T21:38:27.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:27 smithi181 ceph-mon[25811]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:27.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:27 smithi181 ceph-mon[25811]: osd.6 [v2:172.21.15.181:6816/1179366902,v1:172.21.15.181:6817/1179366902] boot 2024-09-13T21:38:27.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:27 smithi181 ceph-mon[25811]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T21:38:27.363 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:27.363 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:27 smithi181 ceph-mon[25811]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T21:38:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:27 smithi154 ceph-mon[23700]: purged_snaps scrub starts 2024-09-13T21:38:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:27 smithi154 ceph-mon[23700]: purged_snaps scrub ok 2024-09-13T21:38:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:27 smithi154 ceph-mon[23700]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T21:38:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:27 smithi154 ceph-mon[23700]: osd.6 [v2:172.21.15.181:6816/1179366902,v1:172.21.15.181:6817/1179366902] boot 2024-09-13T21:38:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:27 smithi154 ceph-mon[23700]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T21:38:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:38:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:27 smithi154 ceph-mon[23700]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T21:38:29.368 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:29 smithi181 ceph-mon[25811]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:29.368 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:29 smithi181 ceph-mon[25811]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T21:38:29.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:29 smithi154 ceph-mon[23700]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:29.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:29 smithi154 ceph-mon[23700]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T21:38:30.386 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T21:38:30.386 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: Detected new or changed devices on smithi181 2024-09-13T21:38:30.386 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi181 to 5311M 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:30.387 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:30.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T21:38:30.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: Detected new or changed devices on smithi181 2024-09-13T21:38:30.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:30.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi181 to 5311M 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:30.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:31.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:31 smithi181 ceph-mon[25811]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:31.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:31 smithi181 ceph-mon[25811]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T21:38:31.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:31 smithi181 ceph-mon[25811]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:38:31.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:38:31.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:38:31.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:31.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:31 smithi154 ceph-mon[23700]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:31.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:31 smithi154 ceph-mon[23700]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T21:38:31.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:31 smithi154 ceph-mon[23700]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:38:31.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T21:38:31.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T21:38:31.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:33.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:33 smithi154 ceph-mon[23700]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-13T21:38:33.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:33 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/797973288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50063a63-b25c-4ee8-a2eb-74e11d36b2f1"}]: dispatch 2024-09-13T21:38:33.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:33 smithi154 ceph-mon[23700]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50063a63-b25c-4ee8-a2eb-74e11d36b2f1"}]: dispatch 2024-09-13T21:38:33.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:33 smithi154 ceph-mon[23700]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50063a63-b25c-4ee8-a2eb-74e11d36b2f1"}]': finished 2024-09-13T21:38:33.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:33 smithi154 ceph-mon[23700]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T21:38:33.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:33 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:33 smithi181 ceph-mon[25811]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-13T21:38:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:33 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/797973288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50063a63-b25c-4ee8-a2eb-74e11d36b2f1"}]: dispatch 2024-09-13T21:38:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:33 smithi181 ceph-mon[25811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50063a63-b25c-4ee8-a2eb-74e11d36b2f1"}]: dispatch 2024-09-13T21:38:33.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:33 smithi181 ceph-mon[25811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50063a63-b25c-4ee8-a2eb-74e11d36b2f1"}]': finished 2024-09-13T21:38:33.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:33 smithi181 ceph-mon[25811]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T21:38:33.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:33 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:34.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:34 smithi181 ceph-mon[25811]: from='client.? 172.21.15.181:0/427644628' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:38:34.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:34 smithi154 ceph-mon[23700]: from='client.? 172.21.15.181:0/427644628' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T21:38:35.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:35 smithi154 ceph-mon[23700]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-13T21:38:35.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:35 smithi181 ceph-mon[25811]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-13T21:38:37.267 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:37 smithi181 ceph-mon[25811]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T21:38:37.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:37 smithi154 ceph-mon[23700]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T21:38:39.286 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:39 smithi181 ceph-mon[25811]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:38:39.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:39 smithi154 ceph-mon[23700]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:38:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:38:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:40 smithi181 ceph-mon[25811]: Deploying daemon osd.7 on smithi181 2024-09-13T21:38:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:38:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:40.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:40 smithi154 ceph-mon[23700]: Deploying daemon osd.7 on smithi181 2024-09-13T21:38:41.376 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:41 smithi181 ceph-mon[25811]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:38:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:41 smithi154 ceph-mon[23700]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T21:38:43.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:43 smithi154 ceph-mon[23700]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:43 smithi181 ceph-mon[25811]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:45.832 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:45 smithi181 ceph-mon[25811]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:45.832 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:45.832 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:45.832 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:45.832 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:45.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:45 smithi154 ceph-mon[23700]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:45.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:45.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:45.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:45.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:46.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:38:46.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:38:46.898 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:38:46.898 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:38:47.532 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 7 on host 'smithi181' 2024-09-13T21:38:47.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:47 smithi181 ceph-mon[25811]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:47.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:47 smithi154 ceph-mon[23700]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:48.111 DEBUG:teuthology.orchestra.run.smithi181:osd.7> sudo journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T21:38:48.114 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T21:38:48.114 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T21:38:48.623 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:48 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:48.623 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:48 smithi181 ceph-mon[25811]: from='osd.7 [v2:172.21.15.181:6824/3739650659,v1:172.21.15.181:6825/3739650659]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:38:48.623 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:48 smithi181 ceph-mon[25811]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:38:48.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:48 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:48.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:48 smithi154 ceph-mon[23700]: from='osd.7 [v2:172.21.15.181:6824/3739650659,v1:172.21.15.181:6825/3739650659]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:38:48.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:48 smithi154 ceph-mon[23700]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:38:49.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:49 smithi154 ceph-mon[23700]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:49.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:49 smithi154 ceph-mon[23700]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T21:38:49.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:49 smithi154 ceph-mon[23700]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T21:38:49.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:49 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:49.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:49 smithi154 ceph-mon[23700]: from='osd.7 [v2:172.21.15.181:6824/3739650659,v1:172.21.15.181:6825/3739650659]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:49.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:49 smithi154 ceph-mon[23700]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:49.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:49 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:49.761 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-mon[25811]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:49.761 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-mon[25811]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T21:38:49.761 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-mon[25811]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T21:38:49.761 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:49.761 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-mon[25811]: from='osd.7 [v2:172.21.15.181:6824/3739650659,v1:172.21.15.181:6825/3739650659]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:49.761 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-mon[25811]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:38:49.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:49.762 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[39058]: 2024-09-13T21:38:49.541+0000 7f4771dc5700 -1 osd.7 0 waiting for initial osdmap 2024-09-13T21:38:49.762 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:38:49 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[39058]: 2024-09-13T21:38:49.548+0000 7f476bf5b700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:38:50.097 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:38:50.682 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:50 smithi181 ceph-mon[25811]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-13T21:38:50.682 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:50 smithi181 ceph-mon[25811]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T21:38:50.682 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:50 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:50.682 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:50 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:50.682 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:50 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1371203289' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:38:50.682 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:50 smithi181 ceph-mon[25811]: osd.7 [v2:172.21.15.181:6824/3739650659,v1:172.21.15.181:6825/3739650659] boot 2024-09-13T21:38:50.682 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:50 smithi181 ceph-mon[25811]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T21:38:50.682 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:50 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:50.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:50 smithi154 ceph-mon[23700]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-09-13T21:38:50.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:50 smithi154 ceph-mon[23700]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T21:38:50.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:50 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:50.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:50 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:50.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:50 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1371203289' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:38:50.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:50 smithi154 ceph-mon[23700]: osd.7 [v2:172.21.15.181:6824/3739650659,v1:172.21.15.181:6825/3739650659] boot 2024-09-13T21:38:50.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:50 smithi154 ceph-mon[23700]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T21:38:50.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:50 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:38:50.768 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1726263506,"num_in_osds":8,"osd_in_since":1726263513,"num_remapped_pgs":0} 2024-09-13T21:38:51.769 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T21:38:51.796 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:51 smithi154 ceph-mon[23700]: purged_snaps scrub starts 2024-09-13T21:38:51.796 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:51 smithi154 ceph-mon[23700]: purged_snaps scrub ok 2024-09-13T21:38:51.796 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:51 smithi154 ceph-mon[23700]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:51.796 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:51 smithi154 ceph-mon[23700]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T21:38:51.830 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:51 smithi181 ceph-mon[25811]: purged_snaps scrub starts 2024-09-13T21:38:51.830 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:51 smithi181 ceph-mon[25811]: purged_snaps scrub ok 2024-09-13T21:38:51.830 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:51 smithi181 ceph-mon[25811]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T21:38:51.830 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:51 smithi181 ceph-mon[25811]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T21:38:53.364 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:38:53.364 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: Detected new or changed devices on smithi181 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi181 to 3983M 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:53.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:53.366 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:53.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:38:53.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: Detected new or changed devices on smithi181 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi181 to 3983M 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:38:53.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:38:53.842 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:38:54.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:54 smithi154 ceph-mon[23700]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T21:38:54.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:54 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2185647921' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:38:54.506 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1726263530,"num_in_osds":8,"osd_in_since":1726263513,"num_remapped_pgs":0} 2024-09-13T21:38:54.507 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T21:38:54.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:54 smithi181 ceph-mon[25811]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T21:38:54.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:54 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2185647921' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T21:38:54.753 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:38:55.548 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:55 smithi154 ceph-mon[23700]: pgmap v170: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:38:55.548 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:55 smithi154 ceph-mon[23700]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T21:38:55.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:55 smithi181 ceph-mon[25811]: pgmap v170: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T21:38:55.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:55 smithi181 ceph-mon[25811]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T21:38:56.513 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:38:56.513 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":59,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","created":"2024-09-13T21:33:00.570048+0000","modified":"2024-09-13T21:38:54.188563+0000","last_up_change":"2024-09-13T21:38:50.127154+0000","last_in_change":"2024-09-13T21:38:33.124661+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T21:36:52.386474+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fde9e1d4-1248-4c90-8ec2-9266a3d387b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6803","nonce":3023533527}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6805","nonce":3023533527}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6809","nonce":3023533527}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6807","nonce":3023533527}]},"public_addr":"172.21.15.154:6803/3023533527","cluster_addr":"172.21.15.154:6805/3023533527","heartbeat_back_addr":"172.21.15.154:6809/3023533527","heartbeat_front_addr":"172.21.15.154:6807/3023533527","state":["exists","up"]},{"osd":1,"uuid":"63926304-68de-4f03-8384-f04b29d10496","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6811","nonce":3277122724}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6813","nonce":3277122724}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6817","nonce":3277122724}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6815","nonce":3277122724}]},"public_addr":"172.21.15.154:6811/3277122724","cluster_addr":"172.21.15.154:6813/3277122724","heartbeat_back_addr":"172.21.15.154:6817/3277122724","heartbeat_front_addr":"172.21.15.154:6815/3277122724","state":["exists","up"]},{"osd":2,"uuid":"a5061bfe-56e7-4d7d-9999-a97d7fb4b79d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6819","nonce":1362376451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6821","nonce":1362376451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6825","nonce":1362376451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6823","nonce":1362376451}]},"public_addr":"172.21.15.154:6819/1362376451","cluster_addr":"172.21.15.154:6821/1362376451","heartbeat_back_addr":"172.21.15.154:6825/1362376451","heartbeat_front_addr":"172.21.15.154:6823/1362376451","state":["exists","up"]},{"osd":3,"uuid":"cb1e4228-7c42-47a3-a6ee-1f613dfcc829","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6827","nonce":616538664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6829","nonce":616538664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6833","nonce":616538664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6831","nonce":616538664}]},"public_addr":"172.21.15.154:6827/616538664","cluster_addr":"172.21.15.154:6829/616538664","heartbeat_back_addr":"172.21.15.154:6833/616538664","heartbeat_front_addr":"172.21.15.154:6831/616538664","state":["exists","up"]},{"osd":4,"uuid":"b704e20c-943d-42c1-b868-8cd7d317536c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6801","nonce":2957526050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6803","nonce":2957526050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6807","nonce":2957526050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6805","nonce":2957526050}]},"public_addr":"172.21.15.181:6801/2957526050","cluster_addr":"172.21.15.181:6803/2957526050","heartbeat_back_addr":"172.21.15.181:6807/2957526050","heartbeat_front_addr":"172.21.15.181:6805/2957526050","state":["exists","up"]},{"osd":5,"uuid":"1daa049b-e978-47bb-b333-fd41cc4c3885","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6809","nonce":1539599335}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6811","nonce":1539599335}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6815","nonce":1539599335}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6813","nonce":1539599335}]},"public_addr":"172.21.15.181:6809/1539599335","cluster_addr":"172.21.15.181:6811/1539599335","heartbeat_back_addr":"172.21.15.181:6815/1539599335","heartbeat_front_addr":"172.21.15.181:6813/1539599335","state":["exists","up"]},{"osd":6,"uuid":"549ac1b3-8f18-4b1b-b2b6-330115060298","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6817","nonce":1179366902}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6819","nonce":1179366902}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6823","nonce":1179366902}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6821","nonce":1179366902}]},"public_addr":"172.21.15.181:6817/1179366902","cluster_addr":"172.21.15.181:6819/1179366902","heartbeat_back_addr":"172.21.15.181:6823/1179366902","heartbeat_front_addr":"172.21.15.181:6821/1179366902","state":["exists","up"]},{"osd":7,"uuid":"50063a63-b25c-4ee8-a2eb-74e11d36b2f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6825","nonce":3739650659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6827","nonce":3739650659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6831","nonce":3739650659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6829","nonce":3739650659}]},"public_addr":"172.21.15.181:6825/3739650659","cluster_addr":"172.21.15.181:6827/3739650659","heartbeat_back_addr":"172.21.15.181:6831/3739650659","heartbeat_front_addr":"172.21.15.181:6829/3739650659","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:35:54.874769+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:36:20.694097+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:36:49.761882+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:37:16.290880+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:37:37.279963+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:00.470981+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:24.246129+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:48.605257+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.154:0/821862025":"2024-09-14T21:34:46.044627+0000","172.21.15.154:0/2519343460":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6801/809290588":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6800/809290588":"2024-09-14T21:34:46.044627+0000","172.21.15.154:0/2492379091":"2024-09-14T21:33:56.393917+0000","172.21.15.154:6800/1585211596":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/1250917224":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/645167153":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/2224386239":"2024-09-14T21:33:27.435718+0000","172.21.15.154:6800/1047512777":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/1275095755":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6801/1047512777":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/3234687704":"2024-09-14T21:33:56.393917+0000","172.21.15.154:6801/1585211596":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/3291926522":"2024-09-14T21:33:27.435718+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T21:38:57.122 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T21:36:52.386474+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T21:38:57.122 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T21:38:57.329 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:38:57.450 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:57 smithi154 ceph-mon[23700]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-13T21:38:57.451 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:57 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3117781514' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:38:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:57 smithi181 ceph-mon[25811]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-13T21:38:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:57 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/3117781514' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:38:59.223 INFO:teuthology.orchestra.run.smithi154.stdout:pg_num: 1 2024-09-13T21:38:59.517 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:38:59 smithi154 ceph-mon[23700]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T21:38:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:38:59 smithi181 ceph-mon[25811]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T21:38:59.893 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T21:38:59.893 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T21:39:00.470 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:00 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/716825260' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T21:39:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:00 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/716825260' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T21:39:01.498 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:01 smithi154 ceph-mon[23700]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:39:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:01 smithi181 ceph-mon[25811]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T21:39:01.980 INFO:teuthology.orchestra.run.smithi154.stdout:[client.0] 2024-09-13T21:39:01.980 INFO:teuthology.orchestra.run.smithi154.stdout: key = AQD1sORmiqkkOhAAsbXnIHh/JSZ2iJmeWFsCjA== 2024-09-13T21:39:02.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:02 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2201580740' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:39:02.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:02 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2201580740' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T21:39:02.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:02 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2201580740' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T21:39:02.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:02 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2201580740' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T21:39:02.739 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-09-13T21:39:02.739 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T21:39:02.739 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T21:39:02.777 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T21:39:02.777 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T21:39:02.777 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T21:39:03.022 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:04.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:03 smithi154 ceph-mon[23700]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:04.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:03 smithi181 ceph-mon[25811]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:04.869 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:39:05.170 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:04 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3272132069' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T21:39:05.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/3272132069' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T21:39:05.510 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi154.ayrwdi","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":3467215891},{"type":"v1","addr":"172.21.15.154:6801","nonce":3467215891}]},"active_addr":"172.21.15.154:6801/3467215891","active_change":"2024-09-13T21:34:46.044908+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi181.byvkib","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.154:8443/","prometheus":"http://172.21.15.154:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.154:0","nonce":3825681688}]},{"addrvec":[{"type":"v2","addr":"172.21.15.154:0","nonce":947787724}]},{"addrvec":[{"type":"v2","addr":"172.21.15.154:0","nonce":433609406}]},{"addrvec":[{"type":"v2","addr":"172.21.15.154:0","nonce":1143265998}]}]}} 2024-09-13T21:39:05.516 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T21:39:05.517 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T21:39:05.517 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T21:39:05.726 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:06.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:05 smithi154 ceph-mon[23700]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:06.301 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:05 smithi181 ceph-mon[25811]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:07.520 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:39:07.521 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":59,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","created":"2024-09-13T21:33:00.570048+0000","modified":"2024-09-13T21:38:54.188563+0000","last_up_change":"2024-09-13T21:38:50.127154+0000","last_in_change":"2024-09-13T21:38:33.124661+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T21:36:52.386474+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fde9e1d4-1248-4c90-8ec2-9266a3d387b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6803","nonce":3023533527}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6805","nonce":3023533527}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6809","nonce":3023533527}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6807","nonce":3023533527}]},"public_addr":"172.21.15.154:6803/3023533527","cluster_addr":"172.21.15.154:6805/3023533527","heartbeat_back_addr":"172.21.15.154:6809/3023533527","heartbeat_front_addr":"172.21.15.154:6807/3023533527","state":["exists","up"]},{"osd":1,"uuid":"63926304-68de-4f03-8384-f04b29d10496","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6811","nonce":3277122724}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6813","nonce":3277122724}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6817","nonce":3277122724}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6815","nonce":3277122724}]},"public_addr":"172.21.15.154:6811/3277122724","cluster_addr":"172.21.15.154:6813/3277122724","heartbeat_back_addr":"172.21.15.154:6817/3277122724","heartbeat_front_addr":"172.21.15.154:6815/3277122724","state":["exists","up"]},{"osd":2,"uuid":"a5061bfe-56e7-4d7d-9999-a97d7fb4b79d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6819","nonce":1362376451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6821","nonce":1362376451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6825","nonce":1362376451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6823","nonce":1362376451}]},"public_addr":"172.21.15.154:6819/1362376451","cluster_addr":"172.21.15.154:6821/1362376451","heartbeat_back_addr":"172.21.15.154:6825/1362376451","heartbeat_front_addr":"172.21.15.154:6823/1362376451","state":["exists","up"]},{"osd":3,"uuid":"cb1e4228-7c42-47a3-a6ee-1f613dfcc829","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6827","nonce":616538664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6829","nonce":616538664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6833","nonce":616538664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6831","nonce":616538664}]},"public_addr":"172.21.15.154:6827/616538664","cluster_addr":"172.21.15.154:6829/616538664","heartbeat_back_addr":"172.21.15.154:6833/616538664","heartbeat_front_addr":"172.21.15.154:6831/616538664","state":["exists","up"]},{"osd":4,"uuid":"b704e20c-943d-42c1-b868-8cd7d317536c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6801","nonce":2957526050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6803","nonce":2957526050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6807","nonce":2957526050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6805","nonce":2957526050}]},"public_addr":"172.21.15.181:6801/2957526050","cluster_addr":"172.21.15.181:6803/2957526050","heartbeat_back_addr":"172.21.15.181:6807/2957526050","heartbeat_front_addr":"172.21.15.181:6805/2957526050","state":["exists","up"]},{"osd":5,"uuid":"1daa049b-e978-47bb-b333-fd41cc4c3885","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6809","nonce":1539599335}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6811","nonce":1539599335}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6815","nonce":1539599335}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6813","nonce":1539599335}]},"public_addr":"172.21.15.181:6809/1539599335","cluster_addr":"172.21.15.181:6811/1539599335","heartbeat_back_addr":"172.21.15.181:6815/1539599335","heartbeat_front_addr":"172.21.15.181:6813/1539599335","state":["exists","up"]},{"osd":6,"uuid":"549ac1b3-8f18-4b1b-b2b6-330115060298","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6817","nonce":1179366902}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6819","nonce":1179366902}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6823","nonce":1179366902}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6821","nonce":1179366902}]},"public_addr":"172.21.15.181:6817/1179366902","cluster_addr":"172.21.15.181:6819/1179366902","heartbeat_back_addr":"172.21.15.181:6823/1179366902","heartbeat_front_addr":"172.21.15.181:6821/1179366902","state":["exists","up"]},{"osd":7,"uuid":"50063a63-b25c-4ee8-a2eb-74e11d36b2f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6825","nonce":3739650659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6827","nonce":3739650659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6831","nonce":3739650659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6829","nonce":3739650659}]},"public_addr":"172.21.15.181:6825/3739650659","cluster_addr":"172.21.15.181:6827/3739650659","heartbeat_back_addr":"172.21.15.181:6831/3739650659","heartbeat_front_addr":"172.21.15.181:6829/3739650659","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:35:54.874769+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:36:20.694097+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:36:49.761882+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:37:16.290880+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:37:37.279963+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:00.470981+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:24.246129+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:48.605257+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.154:0/821862025":"2024-09-14T21:34:46.044627+0000","172.21.15.154:0/2519343460":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6801/809290588":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6800/809290588":"2024-09-14T21:34:46.044627+0000","172.21.15.154:0/2492379091":"2024-09-14T21:33:56.393917+0000","172.21.15.154:6800/1585211596":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/1250917224":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/645167153":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/2224386239":"2024-09-14T21:33:27.435718+0000","172.21.15.154:6800/1047512777":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/1275095755":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6801/1047512777":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/3234687704":"2024-09-14T21:33:56.393917+0000","172.21.15.154:6801/1585211596":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/3291926522":"2024-09-14T21:33:27.435718+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T21:39:08.076 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:07 smithi154 ceph-mon[23700]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:08.076 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:07 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1098795832' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:39:08.099 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T21:39:08.099 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T21:39:08.306 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:07 smithi181 ceph-mon[25811]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:07 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1098795832' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:39:10.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:09 smithi154 ceph-mon[23700]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:10.150 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:39:10.150 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":59,"fsid":"ad78d70a-7217-11ef-bceb-c7b262605968","created":"2024-09-13T21:33:00.570048+0000","modified":"2024-09-13T21:38:54.188563+0000","last_up_change":"2024-09-13T21:38:50.127154+0000","last_in_change":"2024-09-13T21:38:33.124661+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T21:36:52.386474+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fde9e1d4-1248-4c90-8ec2-9266a3d387b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6803","nonce":3023533527}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6805","nonce":3023533527}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6809","nonce":3023533527}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":3023533527},{"type":"v1","addr":"172.21.15.154:6807","nonce":3023533527}]},"public_addr":"172.21.15.154:6803/3023533527","cluster_addr":"172.21.15.154:6805/3023533527","heartbeat_back_addr":"172.21.15.154:6809/3023533527","heartbeat_front_addr":"172.21.15.154:6807/3023533527","state":["exists","up"]},{"osd":1,"uuid":"63926304-68de-4f03-8384-f04b29d10496","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6811","nonce":3277122724}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6813","nonce":3277122724}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6817","nonce":3277122724}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":3277122724},{"type":"v1","addr":"172.21.15.154:6815","nonce":3277122724}]},"public_addr":"172.21.15.154:6811/3277122724","cluster_addr":"172.21.15.154:6813/3277122724","heartbeat_back_addr":"172.21.15.154:6817/3277122724","heartbeat_front_addr":"172.21.15.154:6815/3277122724","state":["exists","up"]},{"osd":2,"uuid":"a5061bfe-56e7-4d7d-9999-a97d7fb4b79d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6819","nonce":1362376451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6821","nonce":1362376451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6825","nonce":1362376451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1362376451},{"type":"v1","addr":"172.21.15.154:6823","nonce":1362376451}]},"public_addr":"172.21.15.154:6819/1362376451","cluster_addr":"172.21.15.154:6821/1362376451","heartbeat_back_addr":"172.21.15.154:6825/1362376451","heartbeat_front_addr":"172.21.15.154:6823/1362376451","state":["exists","up"]},{"osd":3,"uuid":"cb1e4228-7c42-47a3-a6ee-1f613dfcc829","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6827","nonce":616538664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6828","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6829","nonce":616538664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6832","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6833","nonce":616538664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6830","nonce":616538664},{"type":"v1","addr":"172.21.15.154:6831","nonce":616538664}]},"public_addr":"172.21.15.154:6827/616538664","cluster_addr":"172.21.15.154:6829/616538664","heartbeat_back_addr":"172.21.15.154:6833/616538664","heartbeat_front_addr":"172.21.15.154:6831/616538664","state":["exists","up"]},{"osd":4,"uuid":"b704e20c-943d-42c1-b868-8cd7d317536c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6801","nonce":2957526050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6803","nonce":2957526050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6807","nonce":2957526050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":2957526050},{"type":"v1","addr":"172.21.15.181:6805","nonce":2957526050}]},"public_addr":"172.21.15.181:6801/2957526050","cluster_addr":"172.21.15.181:6803/2957526050","heartbeat_back_addr":"172.21.15.181:6807/2957526050","heartbeat_front_addr":"172.21.15.181:6805/2957526050","state":["exists","up"]},{"osd":5,"uuid":"1daa049b-e978-47bb-b333-fd41cc4c3885","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6809","nonce":1539599335}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6811","nonce":1539599335}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6815","nonce":1539599335}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":1539599335},{"type":"v1","addr":"172.21.15.181:6813","nonce":1539599335}]},"public_addr":"172.21.15.181:6809/1539599335","cluster_addr":"172.21.15.181:6811/1539599335","heartbeat_back_addr":"172.21.15.181:6815/1539599335","heartbeat_front_addr":"172.21.15.181:6813/1539599335","state":["exists","up"]},{"osd":6,"uuid":"549ac1b3-8f18-4b1b-b2b6-330115060298","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6817","nonce":1179366902}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6819","nonce":1179366902}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6823","nonce":1179366902}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1179366902},{"type":"v1","addr":"172.21.15.181:6821","nonce":1179366902}]},"public_addr":"172.21.15.181:6817/1179366902","cluster_addr":"172.21.15.181:6819/1179366902","heartbeat_back_addr":"172.21.15.181:6823/1179366902","heartbeat_front_addr":"172.21.15.181:6821/1179366902","state":["exists","up"]},{"osd":7,"uuid":"50063a63-b25c-4ee8-a2eb-74e11d36b2f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6825","nonce":3739650659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6827","nonce":3739650659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6831","nonce":3739650659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":3739650659},{"type":"v1","addr":"172.21.15.181:6829","nonce":3739650659}]},"public_addr":"172.21.15.181:6825/3739650659","cluster_addr":"172.21.15.181:6827/3739650659","heartbeat_back_addr":"172.21.15.181:6831/3739650659","heartbeat_front_addr":"172.21.15.181:6829/3739650659","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:35:54.874769+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:36:20.694097+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:36:49.761882+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:37:16.290880+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:37:37.279963+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:00.470981+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:24.246129+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T21:38:48.605257+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.154:0/821862025":"2024-09-14T21:34:46.044627+0000","172.21.15.154:0/2519343460":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6801/809290588":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6800/809290588":"2024-09-14T21:34:46.044627+0000","172.21.15.154:0/2492379091":"2024-09-14T21:33:56.393917+0000","172.21.15.154:6800/1585211596":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/1250917224":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/645167153":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/2224386239":"2024-09-14T21:33:27.435718+0000","172.21.15.154:6800/1047512777":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/1275095755":"2024-09-14T21:34:46.044627+0000","172.21.15.154:6801/1047512777":"2024-09-14T21:33:27.435718+0000","172.21.15.154:0/3234687704":"2024-09-14T21:33:56.393917+0000","172.21.15.154:6801/1585211596":"2024-09-14T21:33:56.393917+0000","172.21.15.154:0/3291926522":"2024-09-14T21:33:27.435718+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T21:39:10.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:09 smithi181 ceph-mon[25811]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:10.831 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T21:39:10.832 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T21:39:10.832 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T21:39:10.832 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T21:39:10.833 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T21:39:10.833 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T21:39:10.834 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T21:39:10.834 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T21:39:11.287 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:11.288 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:11.288 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:11.288 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:11.289 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:11.289 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:11.291 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:11.293 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:11 smithi181 ceph-mon[25811]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:11 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/3884719129' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:39:11.425 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:11 smithi154 ceph-mon[23700]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:11.425 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:11 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3884719129' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T21:39:13.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:13 smithi154 ceph-mon[23700]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:13.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:13 smithi181 ceph-mon[25811]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:15.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:15 smithi154 ceph-mon[23700]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:15.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:15 smithi181 ceph-mon[25811]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:17.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:17 smithi181 ceph-mon[25811]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:17.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:17 smithi154 ceph-mon[23700]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:19.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:19 smithi154 ceph-mon[23700]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:19.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:19 smithi181 ceph-mon[25811]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:21 smithi181 ceph-mon[25811]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:21.464 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:21 smithi154 ceph-mon[23700]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:22.396 INFO:teuthology.orchestra.run.smithi154.stdout:38654705708 2024-09-13T21:39:22.396 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T21:39:23.294 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:23 smithi154 ceph-mon[23700]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:23.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:23 smithi181 ceph-mon[25811]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:24.183 INFO:teuthology.orchestra.run.smithi154.stdout:111669149724 2024-09-13T21:39:24.183 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T21:39:25.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:25 smithi154 ceph-mon[23700]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:25.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:25 smithi181 ceph-mon[25811]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:25.753 INFO:teuthology.orchestra.run.smithi154.stdout:133143986199 2024-09-13T21:39:25.753 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T21:39:26.220 INFO:teuthology.orchestra.run.smithi154.stdout:60129542183 2024-09-13T21:39:26.220 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T21:39:26.235 INFO:teuthology.orchestra.run.smithi154.stdout:201863462925 2024-09-13T21:39:26.235 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T21:39:26.846 INFO:teuthology.orchestra.run.smithi154.stdout:81604378657 2024-09-13T21:39:26.847 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T21:39:27.042 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:27.444 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:27 smithi154 ceph-mon[23700]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:27.467 INFO:teuthology.orchestra.run.smithi154.stdout:236223201289 2024-09-13T21:39:27.467 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T21:39:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:27 smithi181 ceph-mon[25811]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:27.744 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:27.744 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:27.745 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:29.019 INFO:teuthology.orchestra.run.smithi154.stdout:167503724564 2024-09-13T21:39:29.020 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T21:39:29.282 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:29 smithi154 ceph-mon[23700]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:29 smithi181 ceph-mon[25811]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:30.898 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:30.898 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:31.292 INFO:teuthology.orchestra.run.smithi154.stdout:38654705710 2024-09-13T21:39:31.571 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:31 smithi154 ceph-mon[23700]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:31.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:31 smithi181 ceph-mon[25811]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:32.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:32 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2550816784' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T21:39:32.581 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705710 for osd.0 2024-09-13T21:39:32.582 DEBUG:teuthology.parallel:result is None 2024-09-13T21:39:32.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:32 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2550816784' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T21:39:33.453 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:33 smithi154 ceph-mon[23700]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:33 smithi181 ceph-mon[25811]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:35.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:35 smithi154 ceph-mon[23700]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:35.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:35 smithi181 ceph-mon[25811]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:35.918 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:35.918 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:36.275 INFO:teuthology.orchestra.run.smithi154.stdout:236223201292 2024-09-13T21:39:37.453 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:37 smithi154 ceph-mon[23700]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:37.453 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:37 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1136539719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T21:39:37.502 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201292 for osd.7 2024-09-13T21:39:37.503 DEBUG:teuthology.parallel:result is None 2024-09-13T21:39:37.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:37 smithi181 ceph-mon[25811]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:37.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:37 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1136539719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T21:39:37.861 INFO:teuthology.orchestra.run.smithi154.stdout:111669149727 2024-09-13T21:39:38.399 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:38 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/567272839' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T21:39:38.425 INFO:teuthology.orchestra.run.smithi154.stdout:81604378660 2024-09-13T21:39:38.627 INFO:teuthology.orchestra.run.smithi154.stdout:60129542186 2024-09-13T21:39:38.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:38 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/567272839' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T21:39:39.453 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:39 smithi154 ceph-mon[23700]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:39.453 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:39 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/646815591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T21:39:39.453 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:39 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2673494072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T21:39:39.551 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149727 for osd.3 2024-09-13T21:39:39.552 DEBUG:teuthology.parallel:result is None 2024-09-13T21:39:39.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:39 smithi181 ceph-mon[25811]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:39.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:39 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/646815591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T21:39:39.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:39 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2673494072' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T21:39:40.466 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378660 for osd.2 2024-09-13T21:39:40.466 DEBUG:teuthology.parallel:result is None 2024-09-13T21:39:41.079 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542186 for osd.1 2024-09-13T21:39:41.080 DEBUG:teuthology.parallel:result is None 2024-09-13T21:39:41.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:41 smithi154 ceph-mon[23700]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:41 smithi181 ceph-mon[25811]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:42.887 INFO:teuthology.orchestra.run.smithi154.stdout:167503724567 2024-09-13T21:39:43.298 INFO:teuthology.orchestra.run.smithi154.stdout:133143986203 2024-09-13T21:39:43.535 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:43 smithi154 ceph-mon[23700]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:43.536 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:43 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1716752847' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T21:39:43.536 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:43 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3121757705' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T21:39:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:43 smithi181 ceph-mon[25811]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:43 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1716752847' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T21:39:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:43 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/3121757705' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T21:39:43.846 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986199 got 133143986203 for osd.4 2024-09-13T21:39:43.846 DEBUG:teuthology.parallel:result is None 2024-09-13T21:39:44.209 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724564 got 167503724567 for osd.5 2024-09-13T21:39:44.209 DEBUG:teuthology.parallel:result is None 2024-09-13T21:39:44.721 INFO:teuthology.orchestra.run.smithi154.stdout:201863462930 2024-09-13T21:39:45.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:45 smithi154 ceph-mon[23700]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:45.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:45 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2319494183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T21:39:45.410 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462925 got 201863462930 for osd.6 2024-09-13T21:39:45.410 DEBUG:teuthology.parallel:result is None 2024-09-13T21:39:45.410 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T21:39:45.411 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T21:39:45.620 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:45.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:45 smithi181 ceph-mon[25811]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:45.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:45 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2319494183' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T21:39:46.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:39:46.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:39:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:39:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:39:47.255 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:47 smithi154 ceph-mon[23700]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:47.430 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:39:47.432 INFO:teuthology.orchestra.run.smithi154.stderr:dumped all 2024-09-13T21:39:47.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:47 smithi181 ceph-mon[25811]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:48.052 INFO:teuthology.orchestra.run.smithi154.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-09-13T21:39:46.133626+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48868,"kb_used_data":6308,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912348,"statfs":{"total":767960285184,"available":767910244352,"internally_reserved":0,"allocated":6459392,"data_stored":3546753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002544"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T21:38:54.202152+0000","last_change":"2024-09-13T21:38:54.202152+0000","last_active":"2024-09-13T21:38:54.202152+0000","last_peered":"2024-09-13T21:38:54.202152+0000","last_clean":"2024-09-13T21:38:54.202152+0000","last_became_active":"2024-09-13T21:38:54.201672+0000","last_became_peered":"2024-09-13T21:38:54.201672+0000","last_unstale":"2024-09-13T21:38:54.202152+0000","last_undegraded":"2024-09-13T21:38:54.202152+0000","last_fullsized":"2024-09-13T21:38:54.202152+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T21:36:52.602084+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T21:36:52.602084+0000","last_clean_scrub_stamp":"2024-09-13T21:36:52.602084+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:51:01.841270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.499}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48799999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.58699999999999997}]},{"osd":1,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.64500000000000002}]},{"osd":2,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.53200000000000003}]},{"osd":3,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.624}]},{"osd":4,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.435}]},{"osd":5,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.69399999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:39:28 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.32900000000000001,"15min":0.30499999999999999},"min":{"1min":0.26700000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.33700000000000002,"15min":0.30499999999999999},"min":{"1min":0.26300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.65000000000000002}]},{"osd":2,"last update":"Fri Sep 13 21:38:57 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.373,"15min":0.372},"min":{"1min":0.29999999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.497,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39900000000000002,"15min":0.39800000000000002},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.622,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.53300000000000003}]},{"osd":3,"last update":"Fri Sep 13 21:39:22 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.44,"15min":0.44},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.41899999999999998,"15min":0.42099999999999999},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.59199999999999997}]},{"osd":4,"last update":"Fri Sep 13 21:38:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.751}]},{"osd":5,"last update":"Fri Sep 13 21:39:03 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.78300000000000003}]},{"osd":6,"last update":"Fri Sep 13 21:39:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.79400000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80700000000000005}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 21:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.33600000000000002,"15min":0.318},"min":{"1min":0.216,"5min":0.19,"15min":0.19},"max":{"1min":0.51600000000000001,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.35399999999999998,"15min":0.33400000000000002},"min":{"1min":0.26200000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53100000000000003,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.28999999999999998}]},{"osd":2,"last update":"Fri Sep 13 21:38:58 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.36299999999999999,"15min":0.35399999999999998},"min":{"1min":0.314,"5min":0.246,"15min":0.246},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.36599999999999999,"15min":0.35399999999999998},"min":{"1min":0.28999999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.34100000000000003}]},{"osd":3,"last update":"Fri Sep 13 21:39:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.56699999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.379},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29699999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.53800000000000003,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.40600000000000003}]},{"osd":4,"last update":"Fri Sep 13 21:38:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.371},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.42799999999999999}]},{"osd":5,"last update":"Fri Sep 13 21:39:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.44}]},{"osd":6,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.56599999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6232,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":606208,"data_stored":244572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:38:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.36699999999999999,"15min":0.35899999999999999},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.437,"5min":0.373,"15min":0.36299999999999999},"min":{"1min":0.26200000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.38900000000000001}]},{"osd":1,"last update":"Fri Sep 13 21:38:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.377,"15min":0.371},"min":{"1min":0.28399999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.439,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.50600000000000001}]},{"osd":3,"last update":"Fri Sep 13 21:39:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.44500000000000001,"15min":0.45500000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.501,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.432},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.46300000000000002,"15min":0.46899999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.52800000000000002,"5min":0.625,"15min":0.625},"last":0.52000000000000002}]},{"osd":4,"last update":"Fri Sep 13 21:38:41 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.70399999999999996}]},{"osd":5,"last update":"Fri Sep 13 21:39:08 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.69099999999999995}]},{"osd":6,"last update":"Fri Sep 13 21:39:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.94299999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.77700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96899999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:39:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36299999999999999,"15min":0.36199999999999999},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.191},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.379,"5min":0.38800000000000001,"15min":0.39000000000000001},"min":{"1min":0.24399999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.371}]},{"osd":1,"last update":"Fri Sep 13 21:39:20 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40000000000000002,"15min":0.40400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.53700000000000003,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.40699999999999997},"min":{"1min":0.28599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Sep 13 21:39:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.317,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.441,"15min":0.44700000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.61899999999999999,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.47199999999999998}]},{"osd":4,"last update":"Fri Sep 13 21:39:44 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.68000000000000005,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.42399999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.54700000000000004}]},{"osd":5,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.57899999999999996}]},{"osd":6,"last update":"Fri Sep 13 21:39:26 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.503},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53700000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.54600000000000004}]},{"osd":1,"last update":"Fri Sep 13 21:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.628},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.67400000000000004}]},{"osd":2,"last update":"Fri Sep 13 21:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.63800000000000001}]},{"osd":3,"last update":"Fri Sep 13 21:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.55400000000000005}]},{"osd":5,"last update":"Fri Sep 13 21:39:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.36299999999999999}]},{"osd":6,"last update":"Fri Sep 13 21:39:26 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.46800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39200000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.51700000000000002}]},{"osd":1,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":15.057,"5min":15.057,"15min":15.057},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.45400000000000001}]},{"osd":2,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":15.032999999999999,"5min":15.032999999999999,"15min":15.032999999999999},"last":0.48699999999999999}]},{"osd":3,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.625}]},{"osd":4,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.59199999999999997}]},{"osd":6,"last update":"Fri Sep 13 21:39:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.64300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T21:39:48.053 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T21:39:48.269 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:48.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:48 smithi154 ceph-mon[23700]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:39:48.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:48 smithi181 ceph-mon[25811]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:39:49.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:49 smithi154 ceph-mon[23700]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:49.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:49 smithi181 ceph-mon[25811]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:49.993 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:39:49.995 INFO:teuthology.orchestra.run.smithi154.stderr:dumped all 2024-09-13T21:39:50.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:50 smithi154 ceph-mon[23700]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:39:50.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:50 smithi181 ceph-mon[25811]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:39:50.703 INFO:teuthology.orchestra.run.smithi154.stdout:{"pg_ready":true,"pg_map":{"version":198,"stamp":"2024-09-13T21:39:48.133912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48868,"kb_used_data":6308,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912348,"statfs":{"total":767960285184,"available":767910244352,"internally_reserved":0,"allocated":6459392,"data_stored":3546753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002388"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T21:38:54.202152+0000","last_change":"2024-09-13T21:38:54.202152+0000","last_active":"2024-09-13T21:38:54.202152+0000","last_peered":"2024-09-13T21:38:54.202152+0000","last_clean":"2024-09-13T21:38:54.202152+0000","last_became_active":"2024-09-13T21:38:54.201672+0000","last_became_peered":"2024-09-13T21:38:54.201672+0000","last_unstale":"2024-09-13T21:38:54.202152+0000","last_undegraded":"2024-09-13T21:38:54.202152+0000","last_fullsized":"2024-09-13T21:38:54.202152+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T21:36:52.602084+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T21:36:52.602084+0000","last_clean_scrub_stamp":"2024-09-13T21:36:52.602084+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:51:01.841270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.499}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48799999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.434}]},{"osd":1,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.46400000000000002}]},{"osd":2,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.48799999999999999}]},{"osd":3,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.44700000000000001}]},{"osd":4,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.35999999999999999}]},{"osd":5,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.54200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:39:28 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.32900000000000001,"15min":0.30499999999999999},"min":{"1min":0.26700000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.33700000000000002,"15min":0.30499999999999999},"min":{"1min":0.26300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.65000000000000002}]},{"osd":2,"last update":"Fri Sep 13 21:38:57 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.373,"15min":0.372},"min":{"1min":0.29999999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.497,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39900000000000002,"15min":0.39800000000000002},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.622,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.53300000000000003}]},{"osd":3,"last update":"Fri Sep 13 21:39:22 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.44,"15min":0.44},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.41899999999999998,"15min":0.42099999999999999},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.59199999999999997}]},{"osd":4,"last update":"Fri Sep 13 21:38:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.751}]},{"osd":5,"last update":"Fri Sep 13 21:39:03 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.78300000000000003}]},{"osd":6,"last update":"Fri Sep 13 21:39:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.79400000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80700000000000005}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 21:39:33 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.33600000000000002,"15min":0.318},"min":{"1min":0.216,"5min":0.19,"15min":0.19},"max":{"1min":0.51600000000000001,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.35399999999999998,"15min":0.33400000000000002},"min":{"1min":0.26200000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53100000000000003,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.40200000000000002}]},{"osd":2,"last update":"Fri Sep 13 21:38:58 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.36299999999999999,"15min":0.35399999999999998},"min":{"1min":0.314,"5min":0.246,"15min":0.246},"max":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.36599999999999999,"15min":0.35399999999999998},"min":{"1min":0.28999999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.439}]},{"osd":3,"last update":"Fri Sep 13 21:39:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.56699999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.318},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29699999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.53800000000000003,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.372}]},{"osd":4,"last update":"Fri Sep 13 21:39:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.55500000000000005,"15min":0.55000000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53800000000000003}]},{"osd":5,"last update":"Fri Sep 13 21:39:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.55900000000000005}]},{"osd":6,"last update":"Fri Sep 13 21:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.63300000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52700000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6232,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":606208,"data_stored":244572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:38:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.36699999999999999,"15min":0.35899999999999999},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.437,"5min":0.373,"15min":0.36299999999999999},"min":{"1min":0.26200000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.32400000000000001}]},{"osd":1,"last update":"Fri Sep 13 21:38:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.377,"15min":0.371},"min":{"1min":0.28399999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.29499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.34000000000000002}]},{"osd":3,"last update":"Fri Sep 13 21:39:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.44500000000000001,"15min":0.45500000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.501,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.46300000000000002,"15min":0.46899999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.52800000000000002,"5min":0.625,"15min":0.625},"last":0.41399999999999998}]},{"osd":4,"last update":"Fri Sep 13 21:39:42 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.501},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.60299999999999998,"15min":0.60399999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.71299999999999997,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.39700000000000002}]},{"osd":5,"last update":"Fri Sep 13 21:39:08 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.40300000000000002}]},{"osd":6,"last update":"Fri Sep 13 21:39:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.42599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:39:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.36299999999999999,"15min":0.36199999999999999},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.191},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.379,"5min":0.38800000000000001,"15min":0.39000000000000001},"min":{"1min":0.24399999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.371}]},{"osd":1,"last update":"Fri Sep 13 21:39:20 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40000000000000002,"15min":0.40400000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.53700000000000003,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.40699999999999997},"min":{"1min":0.28599999999999998,"5min":0.254,"15min":0.254},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.34300000000000003}]},{"osd":2,"last update":"Fri Sep 13 21:39:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.317,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.441,"15min":0.44700000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.61899999999999999,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.47199999999999998}]},{"osd":4,"last update":"Fri Sep 13 21:39:44 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.68000000000000005,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.42399999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.54700000000000004}]},{"osd":5,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.57899999999999996}]},{"osd":6,"last update":"Fri Sep 13 21:39:26 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.503},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.61399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53700000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.54600000000000004}]},{"osd":1,"last update":"Fri Sep 13 21:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.628},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.67400000000000004}]},{"osd":2,"last update":"Fri Sep 13 21:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.63800000000000001}]},{"osd":3,"last update":"Fri Sep 13 21:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.55400000000000005}]},{"osd":5,"last update":"Fri Sep 13 21:39:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.36299999999999999}]},{"osd":6,"last update":"Fri Sep 13 21:39:26 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.46800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39200000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.51700000000000002}]},{"osd":1,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":15.057,"5min":15.057,"15min":15.057},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.45400000000000001}]},{"osd":2,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":15.032999999999999,"5min":15.032999999999999,"15min":15.032999999999999},"last":0.48699999999999999}]},{"osd":3,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.625}]},{"osd":4,"last update":"Fri Sep 13 21:39:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.59199999999999997}]},{"osd":6,"last update":"Fri Sep 13 21:39:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.64300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T21:39:50.705 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T21:39:50.705 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T21:39:50.705 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T21:39:50.705 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T21:39:50.916 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/mon.smithi154/config 2024-09-13T21:39:51.287 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:51 smithi154 ceph-mon[23700]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:51 smithi181 ceph-mon[25811]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:52.312 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:39:52.312 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:39:52.312 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:52 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:39:52.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:52 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:39:52.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:52 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:39:52.491 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:52 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:39:52.785 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:39:52.785 INFO:teuthology.orchestra.run.smithi154.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T21:39:53.381 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T21:39:53.382 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T21:39:53.382 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T21:39:53.393 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:39:53.393 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-13T21:39:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:53 smithi181 ceph-mon[25811]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:53 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:39:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:53 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2256036068' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T21:39:53.902 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:53 smithi154 ceph-mon[23700]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:53.902 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:53 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:39:53.902 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:53 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2256036068' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T21:39:55.246 INFO:teuthology.orchestra.run.smithi154.stdout:Backend: cephadm 2024-09-13T21:39:55.246 INFO:teuthology.orchestra.run.smithi154.stdout:Available: Yes 2024-09-13T21:39:55.246 INFO:teuthology.orchestra.run.smithi154.stdout:Paused: No 2024-09-13T21:39:55.819 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-13T21:39:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:55 smithi181 ceph-mon[25811]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:56.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:55 smithi154 ceph-mon[23700]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:56.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:56 smithi154 ceph-mon[23700]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:39:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:56 smithi181 ceph-mon[25811]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c 4c0bd657c209 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (5m) 68s ago 5m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (3m) 2m ago 5m 42.7M - 8.3.5 dad864ee21e9 b275b298ebe6 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e c729166e328e 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283 running (4m) 68s ago 4m 412M - 17.2.0 e1d6a67b021e d2a641126ac6 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (6m) 2m ago 6m 42.8M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (4m) 68s ago 4m 39.9M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (5m) 2m ago 5m 17.2M - 1dbe0e931976 8ff8ec26e7ae 2024-09-13T21:39:57.741 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (4m) 68s ago 4m 16.6M - 1dbe0e931976 31f26a6812e0 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (4m) 2m ago 4m 34.1M 3216M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (3m) 2m ago 3m 35.0M 3216M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (3m) 2m ago 3m 32.1M 3216M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (2m) 2m ago 2m 27.8M 3216M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (2m) 68s ago 2m 32.6M 3983M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (2m) 68s ago 2m 33.0M 3983M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (98s) 68s ago 97s 30.4M 3983M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (73s) 68s ago 73s 10.9M 3983M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:39:57.742 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (4m) 2m ago 4m 50.3M - 514e6a882f6e ca18cf24d925 2024-09-13T21:39:58.034 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:57 smithi154 ceph-mon[23700]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:57 smithi181 ceph-mon[25811]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:39:58.366 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-13T21:39:58.963 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:58 smithi154 ceph-mon[23700]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:39:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:58 smithi181 ceph-mon[25811]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:39:59.821 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:39:59 smithi154 ceph-mon[23700]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:39:59 smithi181 ceph-mon[25811]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:00.263 INFO:teuthology.orchestra.run.smithi154.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-13T21:40:00.264 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-13T21:40:00.264 INFO:teuthology.orchestra.run.smithi154.stdout:crash 2/2 2m ago 6m * 2024-09-13T21:40:00.264 INFO:teuthology.orchestra.run.smithi154.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-13T21:40:00.264 INFO:teuthology.orchestra.run.smithi154.stdout:mgr 2/2 2m ago 6m count:2 2024-09-13T21:40:00.264 INFO:teuthology.orchestra.run.smithi154.stdout:mon 2/2 2m ago 5m smithi154:172.21.15.154=smithi154;smithi181:172.21.15.181=smithi181;count:2 2024-09-13T21:40:00.264 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-13T21:40:00.264 INFO:teuthology.orchestra.run.smithi154.stdout:osd 8 2m ago - 2024-09-13T21:40:00.264 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-13T21:40:00.925 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:00 smithi154 ceph-mon[23700]: overall HEALTH_OK 2024-09-13T21:40:00.949 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-13T21:40:01.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:00 smithi181 ceph-mon[25811]: overall HEALTH_OK 2024-09-13T21:40:01.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:01 smithi154 ceph-mon[23700]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:01.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:01 smithi154 ceph-mon[23700]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:01 smithi181 ceph-mon[25811]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:01 smithi181 ceph-mon[25811]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:02.841 INFO:teuthology.orchestra.run.smithi154.stdout:HOST ADDR LABELS STATUS 2024-09-13T21:40:02.841 INFO:teuthology.orchestra.run.smithi154.stdout:smithi154 172.21.15.154 2024-09-13T21:40:02.841 INFO:teuthology.orchestra.run.smithi154.stdout:smithi181 172.21.15.181 2024-09-13T21:40:02.841 INFO:teuthology.orchestra.run.smithi154.stdout:2 hosts in cluster 2024-09-13T21:40:03.494 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-13T21:40:03.792 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:03 smithi154 ceph-mon[23700]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:03.793 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:03 smithi154 ceph-mon[23700]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:03 smithi181 ceph-mon[25811]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:03 smithi181 ceph-mon[25811]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:05.462 INFO:teuthology.orchestra.run.smithi154.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-13T21:40:05.462 INFO:teuthology.orchestra.run.smithi154.stdout:smithi154 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6233003B400BGN 400G LVM detected, locked 2024-09-13T21:40:05.462 INFO:teuthology.orchestra.run.smithi154.stdout:smithi181 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6246000E400BGN 400G LVM detected, locked 2024-09-13T21:40:05.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:05 smithi154 ceph-mon[23700]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:06.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:05 smithi181 ceph-mon[25811]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:06.227 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T21:40:06.238 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:40:06.239 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-13T21:40:06.265 INFO:teuthology.orchestra.run.smithi154.stderr:+ systemctl stop nfs-server 2024-09-13T21:40:06.273 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi181.front.sepia.ceph.com 2024-09-13T21:40:06.273 DEBUG:teuthology.orchestra.run.smithi181:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-13T21:40:06.305 INFO:teuthology.orchestra.run.smithi181.stderr:+ systemctl stop nfs-server 2024-09-13T21:40:06.319 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T21:40:06.328 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:40:06.328 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-13T21:40:06.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:06 smithi154 ceph-mon[23700]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:07.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:06 smithi181 ceph-mon[25811]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:07.834 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:07 smithi154 ceph-mon[23700]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:08.111 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:07 smithi181 ceph-mon[25811]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:09.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:08 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-13T21:40:09.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:08 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-13T21:40:10.059 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:09 smithi154 ceph-mon[23700]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:10.059 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:09 smithi154 ceph-mon[23700]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:10.059 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:09 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-13T21:40:10.059 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:09 smithi154 ceph-mon[23700]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T21:40:10.059 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:09 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-13T21:40:10.059 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:09 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[23696]: 2024-09-13T21:40:09.703+0000 7f37f24c9700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:40:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:09 smithi181 ceph-mon[25811]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:09 smithi181 ceph-mon[25811]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:09 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-13T21:40:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:09 smithi181 ceph-mon[25811]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T21:40:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:09 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-13T21:40:10.381 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-13T21:40:10.391 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-13T21:40:10.391 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T21:40:10.870 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-13T21:40:10.870 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: fsmap foofs:0 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: Saving service mds.foofs spec with placement count:2 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:10.871 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:10.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:10.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi154.uynwpr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:40:10.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi154.uynwpr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T21:40:10.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:10.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:10 smithi154 ceph-mon[23700]: Deploying daemon mds.foofs.smithi154.uynwpr on smithi154 2024-09-13T21:40:11.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-13T21:40:11.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T21:40:11.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-13T21:40:11.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: fsmap foofs:0 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: Saving service mds.foofs spec with placement count:2 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi154.uynwpr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:40:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi154.uynwpr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T21:40:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:10 smithi181 ceph-mon[25811]: Deploying daemon mds.foofs.smithi154.uynwpr on smithi154 2024-09-13T21:40:11.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:11 smithi154 ceph-mon[23700]: pgmap v212: 65 pgs: 10 creating+peering, 1 active+clean, 54 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:11.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:11 smithi154 ceph-mon[23700]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T21:40:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:11 smithi181 ceph-mon[25811]: pgmap v212: 65 pgs: 10 creating+peering, 1 active+clean, 54 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:11 smithi181 ceph-mon[25811]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T21:40:12.987 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:12 smithi154 ceph-mon[23700]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T21:40:13.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:12 smithi181 ceph-mon[25811]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T21:40:13.406 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:40:13.406 INFO:teuthology.orchestra.run.smithi154.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T21:33:45.485648Z", "last_refresh": "2024-09-13T21:37:26.194911Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T21:33:40.086868Z", "last_refresh": "2024-09-13T21:37:26.194975Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T21:33:43.093300Z", "last_refresh": "2024-09-13T21:37:26.195052Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T21:40:09.726569Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T21:40:09.720613Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T21:33:38.169388Z", "last_refresh": "2024-09-13T21:37:26.194841Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi154:172.21.15.154=smithi154", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T21:34:27.330740Z", "last_refresh": "2024-09-13T21:37:26.194686Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T21:33:44.269558Z", "last_refresh": "2024-09-13T21:37:26.195109Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T21:37:26.195225Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T21:33:41.241307Z", "last_refresh": "2024-09-13T21:37:26.195167Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T21:40:14.023 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:13 smithi154 ceph-mon[23700]: pgmap v215: 65 pgs: 20 active+clean, 10 creating+peering, 35 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:13 smithi181 ceph-mon[25811]: pgmap v215: 65 pgs: 20 active+clean, 10 creating+peering, 35 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:14.400 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-13T21:40:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:40:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.mgvpzf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:40:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.mgvpzf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T21:40:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: Deploying daemon mds.foofs.smithi181.mgvpzf on smithi181 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: mds.? [v2:172.21.15.154:6834/3259978861,v1:172.21.15.154:6835/3259978861] up:boot 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: daemon mds.foofs.smithi154.uynwpr assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: Cluster is now healthy 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: fsmap foofs:0 1 up:standby 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:creating} 2024-09-13T21:40:15.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:14 smithi181 ceph-mon[25811]: daemon mds.foofs.smithi154.uynwpr is now active in filesystem foofs as rank 0 2024-09-13T21:40:15.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:40:15.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:15.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.mgvpzf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.mgvpzf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: Deploying daemon mds.foofs.smithi181.mgvpzf on smithi181 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: mds.? [v2:172.21.15.154:6834/3259978861,v1:172.21.15.154:6835/3259978861] up:boot 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: daemon mds.foofs.smithi154.uynwpr assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: Cluster is now healthy 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: fsmap foofs:0 1 up:standby 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:creating} 2024-09-13T21:40:15.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:14 smithi154 ceph-mon[23700]: daemon mds.foofs.smithi154.uynwpr is now active in filesystem foofs as rank 0 2024-09-13T21:40:15.401 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T21:40:16.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:15 smithi154 ceph-mon[23700]: pgmap v216: 65 pgs: 51 active+clean, 4 creating+peering, 10 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:16.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:15 smithi154 ceph-mon[23700]: mds.? [v2:172.21.15.154:6834/3259978861,v1:172.21.15.154:6835/3259978861] up:active 2024-09-13T21:40:16.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:15 smithi154 ceph-mon[23700]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 2024-09-13T21:40:16.250 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:15 smithi181 ceph-mon[25811]: pgmap v216: 65 pgs: 51 active+clean, 4 creating+peering, 10 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:16.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:15 smithi181 ceph-mon[25811]: mds.? [v2:172.21.15.154:6834/3259978861,v1:172.21.15.154:6835/3259978861] up:active 2024-09-13T21:40:16.251 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:15 smithi181 ceph-mon[25811]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 2024-09-13T21:40:17.485 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:40:17.485 INFO:teuthology.orchestra.run.smithi154.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T21:33:45.485648Z", "last_refresh": "2024-09-13T21:37:26.194911Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T21:33:40.086868Z", "last_refresh": "2024-09-13T21:37:26.194975Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T21:33:43.093300Z", "last_refresh": "2024-09-13T21:37:26.195052Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T21:40:09.726569Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T21:40:09.720613Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T21:33:38.169388Z", "last_refresh": "2024-09-13T21:37:26.194841Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi154:172.21.15.154=smithi154", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T21:34:27.330740Z", "last_refresh": "2024-09-13T21:37:26.194686Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T21:33:44.269558Z", "last_refresh": "2024-09-13T21:37:26.195109Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T21:37:26.195225Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T21:33:41.241307Z", "last_refresh": "2024-09-13T21:37:26.195167Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T21:40:17.723 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: pgmap v217: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-09-13T21:40:17.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:17.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:17.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:17.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:17.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:boot 2024-09-13T21:40:17.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:40:17.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:40:17.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:17 smithi181 ceph-mon[25811]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:40:17.812 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: pgmap v217: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-09-13T21:40:17.812 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:17.812 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:17.812 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:17.812 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:17.813 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:boot 2024-09-13T21:40:17.813 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:40:17.813 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:40:17.813 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:17 smithi154 ceph-mon[23700]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:40:18.178 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-13T21:40:18.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:18 smithi181 ceph-mon[25811]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:40:18.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:18 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:18.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:18 smithi154 ceph-mon[23700]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:40:18.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:18 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:19.179 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T21:40:19.517 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:19 smithi154 ceph-mon[23700]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-13T21:40:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:19 smithi181 ceph-mon[25811]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-13T21:40:21.517 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:21 smithi154 ceph-mon[23700]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T21:40:21.517 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:21 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:21.645 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:40:21.645 INFO:teuthology.orchestra.run.smithi154.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T21:33:45.485648Z", "last_refresh": "2024-09-13T21:37:26.194911Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T21:33:40.086868Z", "last_refresh": "2024-09-13T21:37:26.194975Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T21:33:43.093300Z", "last_refresh": "2024-09-13T21:37:26.195052Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T21:40:09.726569Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T21:40:09.720613Z", "last_refresh": "2024-09-13T21:40:18.372407Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T21:33:38.169388Z", "last_refresh": "2024-09-13T21:37:26.194841Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi154:172.21.15.154=smithi154", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T21:34:27.330740Z", "last_refresh": "2024-09-13T21:37:26.194686Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T21:33:44.269558Z", "last_refresh": "2024-09-13T21:37:26.195109Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T21:37:26.195225Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T21:33:41.241307Z", "last_refresh": "2024-09-13T21:37:26.195167Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T21:40:21.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:21 smithi181 ceph-mon[25811]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T21:40:21.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:21 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:22.614 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-13T21:40:22.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:22 smithi154 ceph-mon[23700]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:40:22.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:22 smithi181 ceph-mon[25811]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:40:23.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:23 smithi154 ceph-mon[23700]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-13T21:40:23.615 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T21:40:23.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:23 smithi181 ceph-mon[25811]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-13T21:40:25.696 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:25 smithi154 ceph-mon[23700]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T21:40:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:25 smithi181 ceph-mon[25811]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T21:40:26.128 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:40:26.128 INFO:teuthology.orchestra.run.smithi154.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T21:33:45.485648Z", "last_refresh": "2024-09-13T21:40:25.720852Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T21:33:40.086868Z", "last_refresh": "2024-09-13T21:40:18.371750Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T21:33:43.093300Z", "last_refresh": "2024-09-13T21:40:25.720980Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T21:40:09.726569Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T21:40:09.720613Z", "last_refresh": "2024-09-13T21:40:18.372407Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T21:33:38.169388Z", "last_refresh": "2024-09-13T21:40:18.371930Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi154:172.21.15.154=smithi154", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T21:34:27.330740Z", "last_refresh": "2024-09-13T21:40:18.372006Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T21:33:44.269558Z", "last_refresh": "2024-09-13T21:40:18.372073Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T21:40:18.372139Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T21:33:41.241307Z", "last_refresh": "2024-09-13T21:40:25.721102Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T21:40:26.927 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-13T21:40:26.927 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T21:40:26.936 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:40:26.936 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-13T21:40:27.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:27.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:27.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:27.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:27.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:27.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:26 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:26 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:27.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:27 smithi154 ceph-mon[23700]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:40:27.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:27 smithi154 ceph-mon[23700]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T21:40:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:27 smithi181 ceph-mon[25811]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:40:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:27 smithi181 ceph-mon[25811]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T21:40:30.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:29 smithi154 ceph-mon[23700]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T21:40:30.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:29 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-13T21:40:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:29 smithi181 ceph-mon[25811]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T21:40:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:29 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-13T21:40:30.787 INFO:teuthology.orchestra.run.smithi154.stdout:NFS Cluster Created Successfully 2024-09-13T21:40:31.007 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:30 smithi154 ceph-mon[23700]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:31.007 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-13T21:40:31.007 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:30 smithi154 ceph-mon[23700]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T21:40:31.007 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:30 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-13T21:40:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:30 smithi181 ceph-mon[25811]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-13T21:40:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:30 smithi181 ceph-mon[25811]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T21:40:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:30 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-13T21:40:31.578 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-13T21:40:31.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: pgmap v225: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T21:40:31.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-13T21:40:31.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: Saving service nfs.foo spec with placement count:2 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: Creating key for client.nfs.foo.0.0.smithi154.cmtcgb 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T21:40:31.840 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: Rados config object exists: conf-nfs.foo 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: Creating key for client.nfs.foo.0.0.smithi154.cmtcgb-rgw 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:31.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:31 smithi154 ceph-mon[23700]: Deploying daemon nfs.foo.0.0.smithi154.cmtcgb on smithi154 2024-09-13T21:40:32.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: pgmap v225: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T21:40:32.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: Saving service nfs.foo spec with placement count:2 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: Creating key for client.nfs.foo.0.0.smithi154.cmtcgb 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T21:40:32.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: Rados config object exists: conf-nfs.foo 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: Creating key for client.nfs.foo.0.0.smithi154.cmtcgb-rgw 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:32.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:31 smithi181 ceph-mon[25811]: Deploying daemon nfs.foo.0.0.smithi154.cmtcgb on smithi154 2024-09-13T21:40:32.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:32 smithi154 ceph-mon[23700]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T21:40:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:32 smithi181 ceph-mon[25811]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T21:40:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:33 smithi154 ceph-mon[23700]: pgmap v228: 97 pgs: 29 unknown, 68 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:33 smithi154 ceph-mon[23700]: mgrmap e20: smithi154.ayrwdi(active, since 5m), standbys: smithi181.byvkib 2024-09-13T21:40:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:33 smithi181 ceph-mon[25811]: pgmap v228: 97 pgs: 29 unknown, 68 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:33 smithi181 ceph-mon[25811]: mgrmap e20: smithi154.ayrwdi(active, since 5m), standbys: smithi181.byvkib 2024-09-13T21:40:34.820 INFO:teuthology.orchestra.run.smithi154.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-13T21:40:34.821 INFO:teuthology.orchestra.run.smithi154.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-13T21:40:34.821 INFO:teuthology.orchestra.run.smithi154.stderr:Error EINVAL: invalid command 2024-09-13T21:40:35.190 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:40:35.191 INFO:teuthology.orchestra.run.smithi154.stdout: "bind": "/fake", 2024-09-13T21:40:35.191 INFO:teuthology.orchestra.run.smithi154.stdout: "fs": "foofs", 2024-09-13T21:40:35.191 INFO:teuthology.orchestra.run.smithi154.stdout: "path": "/", 2024-09-13T21:40:35.191 INFO:teuthology.orchestra.run.smithi154.stdout: "cluster": "foo", 2024-09-13T21:40:35.191 INFO:teuthology.orchestra.run.smithi154.stdout: "mode": "RW" 2024-09-13T21:40:35.191 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:40:35.886 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-13T21:40:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: pgmap v229: 97 pgs: 16 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T21:40:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T21:40:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-13T21:40:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T21:40:36.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-13T21:40:36.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T21:40:36.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:35 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: pgmap v229: 97 pgs: 16 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T21:40:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T21:40:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T21:40:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-13T21:40:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T21:40:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-13T21:40:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T21:40:36.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:35 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:37.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:36 smithi154 ceph-mon[23700]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:37.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:36 smithi154 ceph-mon[23700]: Creating key for client.nfs.foo.1.0.smithi181.tvxlgt 2024-09-13T21:40:37.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:36 smithi154 ceph-mon[23700]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T21:40:37.101 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:36 smithi181 ceph-mon[25811]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:37.101 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:36 smithi181 ceph-mon[25811]: Creating key for client.nfs.foo.1.0.smithi181.tvxlgt 2024-09-13T21:40:37.101 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:36 smithi181 ceph-mon[25811]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T21:40:38.052 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:37 smithi154 ceph-mon[23700]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 160 B/s rd, 320 B/s wr, 0 op/s 2024-09-13T21:40:38.138 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:37 smithi181 ceph-mon[25811]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 160 B/s rd, 320 B/s wr, 0 op/s 2024-09-13T21:40:39.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:38 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T21:40:39.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:38 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T21:40:39.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:38 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:40:39.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:38 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:40:39.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:38 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:38 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T21:40:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:38 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T21:40:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:38 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:40:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:38 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T21:40:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:38 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:39 smithi181 ceph-mon[25811]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:39 smithi181 ceph-mon[25811]: Rados config object exists: conf-nfs.foo 2024-09-13T21:40:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:39 smithi181 ceph-mon[25811]: Creating key for client.nfs.foo.1.0.smithi181.tvxlgt-rgw 2024-09-13T21:40:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:39 smithi181 ceph-mon[25811]: Deploying daemon nfs.foo.1.0.smithi181.tvxlgt on smithi181 2024-09-13T21:40:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:39 smithi181 ceph-mon[25811]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 383 B/s wr, 0 op/s 2024-09-13T21:40:40.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:39 smithi154 ceph-mon[23700]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:40.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:39 smithi154 ceph-mon[23700]: Rados config object exists: conf-nfs.foo 2024-09-13T21:40:40.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:39 smithi154 ceph-mon[23700]: Creating key for client.nfs.foo.1.0.smithi181.tvxlgt-rgw 2024-09-13T21:40:40.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:39 smithi154 ceph-mon[23700]: Deploying daemon nfs.foo.1.0.smithi181.tvxlgt on smithi181 2024-09-13T21:40:40.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:39 smithi154 ceph-mon[23700]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 383 B/s wr, 0 op/s 2024-09-13T21:40:40.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:40 smithi154 ceph-mon[23700]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:40.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:40.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:40.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:40.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:40 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:41.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:40 smithi181 ceph-mon[25811]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:41.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:41.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:41.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:41.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:40 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:42.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:41 smithi154 ceph-mon[23700]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-13T21:40:42.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:41 smithi154 ceph-mon[23700]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:42.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:41 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:42.100 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:41 smithi181 ceph-mon[25811]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-13T21:40:42.100 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:41 smithi181 ceph-mon[25811]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:42.100 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:41 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:43.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:42 smithi154 ceph-mon[23700]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:43.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:42 smithi181 ceph-mon[25811]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:43.986 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:43 smithi154 ceph-mon[23700]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-09-13T21:40:43.986 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:43 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:43 smithi181 ceph-mon[25811]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-09-13T21:40:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:43 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:45.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:44 smithi154 ceph-mon[23700]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:44 smithi181 ceph-mon[25811]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:45.966 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-09-13T21:40:46.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-09-13T21:40:46.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:46.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:46.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:46.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:46.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:45 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-09-13T21:40:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:40:46.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:40:46.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:40:46.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:45 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:40:46.736 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T21:40:46.747 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:40:46.747 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-13T21:40:46.775 INFO:teuthology.orchestra.run.smithi154.stderr:+ mkdir /mnt/foo 2024-09-13T21:40:46.786 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-13T21:40:46.850 INFO:teuthology.orchestra.run.smithi154.stderr:++ hostname 2024-09-13T21:40:46.851 INFO:teuthology.orchestra.run.smithi154.stderr:+ mount -t nfs smithi154:/fake /mnt/foo -o sync 2024-09-13T21:40:46.981 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:46 smithi154 ceph-mon[23700]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:46.981 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-13T21:40:46.981 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T21:40:46.982 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-09-13T21:40:46.982 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-09-13T21:40:46.982 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:40:46.982 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:40:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:46 smithi181 ceph-mon[25811]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:40:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-13T21:40:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-09-13T21:40:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-09-13T21:40:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-09-13T21:40:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:40:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:40:47.272 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-13T21:40:47.337 INFO:teuthology.orchestra.run.smithi154.stderr:+ echo test 2024-09-13T21:40:47.347 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-13T21:40:47.411 INFO:teuthology.orchestra.run.smithi154.stderr:+ sync 2024-09-13T21:40:47.646 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T21:40:47.660 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T21:40:47.660 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T21:40:47.661 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T21:40:47.661 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:40:47.662 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T21:40:47.662 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T21:40:47.662 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-13T21:40:47.663 INFO:teuthology.task.exec:Executing custom commands... 2024-09-13T21:40:47.664 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:40:47.664 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-13T21:40:47.731 INFO:teuthology.orchestra.run.smithi154.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-13T21:40:47.731 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:40:47.732 INFO:teuthology.orchestra.run.smithi154.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-13T21:40:47.733 INFO:teuthology.orchestra.run.smithi154.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-13T21:40:47.833 INFO:teuthology.orchestra.run.smithi154.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-13T21:40:47.833 INFO:teuthology.orchestra.run.smithi154.stdout:releasing clients 2024-09-13T21:40:48.005 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:47 smithi154 ceph-mon[23700]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-13T21:40:48.005 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-13T21:40:48.005 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-13T21:40:48.005 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-13T21:40:48.005 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:47 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-13T21:40:48.005 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:47 smithi154 ceph-mon[23700]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T21:40:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:47 smithi181 ceph-mon[25811]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-13T21:40:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:47 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-13T21:40:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:47 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-09-13T21:40:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:47 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-13T21:40:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:47 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-13T21:40:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:47 smithi181 ceph-mon[25811]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T21:40:48.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 130 28.88 MB/sec warmup 1 sec latency 30.688 ms 2024-09-13T21:40:49.023 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:48 smithi154 ceph-mon[23700]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T21:40:49.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:48 smithi181 ceph-mon[25811]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T21:40:49.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 272 25.46 MB/sec warmup 2 sec latency 29.334 ms 2024-09-13T21:40:50.044 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:49 smithi154 ceph-mon[23700]: pgmap v238: 97 pgs: 4 peering, 93 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-09-13T21:40:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:49 smithi181 ceph-mon[25811]: pgmap v238: 97 pgs: 4 peering, 93 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-09-13T21:40:50.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 407 23.97 MB/sec warmup 3 sec latency 34.388 ms 2024-09-13T21:40:51.112 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T21:40:51.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 590 24.89 MB/sec warmup 4 sec latency 24.053 ms 2024-09-13T21:40:51.909 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:51 smithi154 ceph-mon[23700]: pgmap v239: 97 pgs: 4 peering, 93 active+clean; 21 MiB data, 62 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 49 op/s 2024-09-13T21:40:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:51 smithi181 ceph-mon[25811]: pgmap v239: 97 pgs: 4 peering, 93 active+clean; 21 MiB data, 62 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 49 op/s 2024-09-13T21:40:52.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 970 22.07 MB/sec warmup 5 sec latency 23.494 ms 2024-09-13T21:40:53.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 1684 21.84 MB/sec warmup 6 sec latency 22.617 ms 2024-09-13T21:40:53.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:53 smithi154 ceph-mon[23700]: pgmap v240: 97 pgs: 4 peering, 93 active+clean; 42 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.4 MiB/s wr, 283 op/s 2024-09-13T21:40:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:53 smithi181 ceph-mon[25811]: pgmap v240: 97 pgs: 4 peering, 93 active+clean; 42 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.4 MiB/s wr, 283 op/s 2024-09-13T21:40:54.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 2445 20.14 MB/sec warmup 7 sec latency 50.768 ms 2024-09-13T21:40:55.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 3120 21.33 MB/sec warmup 8 sec latency 33.296 ms 2024-09-13T21:40:55.926 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T21:40:55.972 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:55 smithi154 ceph-mon[23700]: pgmap v241: 97 pgs: 97 active+clean; 78 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s wr, 378 op/s 2024-09-13T21:40:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:55 smithi181 ceph-mon[25811]: pgmap v241: 97 pgs: 97 active+clean; 78 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s wr, 378 op/s 2024-09-13T21:40:56.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 3609 21.20 MB/sec warmup 9 sec latency 53.139 ms 2024-09-13T21:40:57.833 INFO:teuthology.orchestra.run.smithi154.stdout: 5 3896 19.97 MB/sec warmup 10 sec latency 36.432 ms 2024-09-13T21:40:58.005 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:57 smithi154 ceph-mon[23700]: pgmap v242: 97 pgs: 97 active+clean; 117 MiB data, 189 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 14 MiB/s wr, 720 op/s 2024-09-13T21:40:58.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:57 smithi181 ceph-mon[25811]: pgmap v242: 97 pgs: 97 active+clean; 117 MiB data, 189 MiB used, 715 GiB / 715 GiB avail; 218 B/s rd, 14 MiB/s wr, 720 op/s 2024-09-13T21:40:58.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 4480 18.63 MB/sec warmup 11 sec latency 43.312 ms 2024-09-13T21:40:59.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 5042 18.59 MB/sec warmup 12 sec latency 43.351 ms 2024-09-13T21:41:00.040 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:40:59 smithi154 ceph-mon[23700]: pgmap v243: 97 pgs: 97 active+clean; 122 MiB data, 205 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 13 MiB/s wr, 759 op/s 2024-09-13T21:41:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:40:59 smithi181 ceph-mon[25811]: pgmap v243: 97 pgs: 97 active+clean; 122 MiB data, 205 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 13 MiB/s wr, 759 op/s 2024-09-13T21:41:00.398 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-13T21:41:00.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 5719 17.71 MB/sec warmup 13 sec latency 77.151 ms 2024-09-13T21:41:01.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 6091 17.27 MB/sec warmup 14 sec latency 32.892 ms 2024-09-13T21:41:01.849 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:01 smithi154 ceph-mon[23700]: pgmap v244: 97 pgs: 97 active+clean; 130 MiB data, 234 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 13 MiB/s wr, 714 op/s 2024-09-13T21:41:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:01 smithi181 ceph-mon[25811]: pgmap v244: 97 pgs: 97 active+clean; 130 MiB data, 234 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 13 MiB/s wr, 714 op/s 2024-09-13T21:41:02.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 6510 17.30 MB/sec warmup 15 sec latency 42.177 ms 2024-09-13T21:41:03.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 7273 18.21 MB/sec warmup 16 sec latency 39.411 ms 2024-09-13T21:41:03.875 INFO:teuthology.orchestra.run.smithi154.stderr:module 'nfs' is already enabled 2024-09-13T21:41:04.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:03 smithi154 ceph-mon[23700]: pgmap v245: 97 pgs: 97 active+clean; 135 MiB data, 267 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 12 MiB/s wr, 791 op/s 2024-09-13T21:41:04.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:03 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/4294934628' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-13T21:41:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:03 smithi181 ceph-mon[25811]: pgmap v245: 97 pgs: 97 active+clean; 135 MiB data, 267 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 12 MiB/s wr, 791 op/s 2024-09-13T21:41:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:03 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/4294934628' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-13T21:41:04.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 7485 17.39 MB/sec warmup 17 sec latency 55.684 ms 2024-09-13T21:41:04.933 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T21:41:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:04 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/4294934628' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-13T21:41:05.106 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:04 smithi154 ceph-mon[23700]: mgrmap e21: smithi154.ayrwdi(active, since 6m), standbys: smithi181.byvkib 2024-09-13T21:41:05.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:04 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/4294934628' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-13T21:41:05.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:04 smithi181 ceph-mon[25811]: mgrmap e21: smithi154.ayrwdi(active, since 6m), standbys: smithi181.byvkib 2024-09-13T21:41:05.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 7871 16.55 MB/sec warmup 18 sec latency 75.146 ms 2024-09-13T21:41:06.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:05 smithi181 ceph-mon[25811]: pgmap v246: 97 pgs: 97 active+clean; 136 MiB data, 276 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 11 MiB/s wr, 700 op/s 2024-09-13T21:41:06.197 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:05 smithi154 ceph-mon[23700]: pgmap v246: 97 pgs: 97 active+clean; 136 MiB data, 276 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 11 MiB/s wr, 700 op/s 2024-09-13T21:41:06.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 8126 15.96 MB/sec warmup 19 sec latency 103.630 ms 2024-09-13T21:41:07.689 INFO:teuthology.orchestra.run.smithi154.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T21:41:07.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 8415 15.62 MB/sec warmup 20 sec latency 63.247 ms 2024-09-13T21:41:08.010 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:07 smithi154 ceph-mon[23700]: pgmap v247: 97 pgs: 97 active+clean; 143 MiB data, 305 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 11 MiB/s wr, 897 op/s 2024-09-13T21:41:08.010 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:08.011 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:41:08.011 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:41:08.011 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:07 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:41:08.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:07 smithi181 ceph-mon[25811]: pgmap v247: 97 pgs: 97 active+clean; 143 MiB data, 305 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 11 MiB/s wr, 897 op/s 2024-09-13T21:41:08.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:08.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:41:08.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:41:08.059 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:07 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:41:08.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 8885 15.37 MB/sec warmup 21 sec latency 81.946 ms 2024-09-13T21:41:08.927 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T21:41:08.928 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:41:08.929 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-13T21:41:09.019 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:08 smithi154 ceph-mon[23700]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:09.020 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:08 smithi154 ceph-mon[23700]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T21:41:09.020 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:08 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:09.020 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:08 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:09.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:08 smithi181 ceph-mon[25811]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:09.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:08 smithi181 ceph-mon[25811]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T21:41:09.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:08 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:09.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:08 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:09.837 INFO:teuthology.orchestra.run.smithi154.stdout: 5 9374 14.90 MB/sec warmup 22 sec latency 41.487 ms 2024-09-13T21:41:09.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:09 smithi154 ceph-mon[23700]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T21:41:09.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:09 smithi154 ceph-mon[23700]: pgmap v248: 97 pgs: 97 active+clean; 145 MiB data, 359 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 605 op/s 2024-09-13T21:41:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:09 smithi181 ceph-mon[25811]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T21:41:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:09 smithi181 ceph-mon[25811]: pgmap v248: 97 pgs: 97 active+clean; 145 MiB data, 359 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 605 op/s 2024-09-13T21:41:10.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 9594 14.51 MB/sec warmup 23 sec latency 39.677 ms 2024-09-13T21:41:11.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 10169 15.01 MB/sec warmup 24 sec latency 38.273 ms 2024-09-13T21:41:11.981 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:11 smithi154 ceph-mon[23700]: pgmap v249: 97 pgs: 97 active+clean; 137 MiB data, 372 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.7 MiB/s wr, 555 op/s 2024-09-13T21:41:12.144 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:41:12.161 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:11 smithi181 ceph-mon[25811]: pgmap v249: 97 pgs: 97 active+clean; 137 MiB data, 372 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.7 MiB/s wr, 555 op/s 2024-09-13T21:41:12.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 10657 15.23 MB/sec warmup 25 sec latency 28.534 ms 2024-09-13T21:41:13.182 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (5m) 27s ago 6m 22.1M - ba2b418f427c 4c0bd657c209 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (6m) 27s ago 6m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (6m) 29s ago 6m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (5m) 27s ago 6m 52.4M - 8.3.5 dad864ee21e9 b275b298ebe6 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (59s) 27s ago 59s 12.6M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (57s) 29s ago 56s 14.7M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:9283 running (8m) 27s ago 8m 460M - 17.2.0 e1d6a67b021e c729166e328e 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283 running (6m) 29s ago 6m 413M - 17.2.0 e1d6a67b021e d2a641126ac6 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (8m) 27s ago 8m 48.4M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (6m) 29s ago 6m 37.9M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (38s) 27s ago 38s 59.9M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:41:13.183 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (32s) 29s ago 32s 46.7M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (6m) 27s ago 6m 17.4M - 1dbe0e931976 8ff8ec26e7ae 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (6m) 29s ago 5m 16.7M - 1dbe0e931976 31f26a6812e0 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (5m) 27s ago 5m 43.2M 3216M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (4m) 27s ago 4m 43.5M 3216M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (4m) 27s ago 4m 40.6M 3216M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (4m) 27s ago 4m 42.0M 3216M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (3m) 29s ago 3m 39.0M 3983M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (3m) 29s ago 3m 39.7M 3983M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (2m) 29s ago 2m 39.3M 3983M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (2m) 29s ago 2m 41.2M 3983M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:41:13.184 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (5m) 27s ago 5m 63.2M - 514e6a882f6e ca18cf24d925 2024-09-13T21:41:13.819 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:41:13.819 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:41:13.819 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:41:13.819 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:41:13.819 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:41:13.819 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:41:13.819 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:41:13.820 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:41:13.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 10914 15.00 MB/sec warmup 26 sec latency 39.763 ms 2024-09-13T21:41:14.011 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:13 smithi154 ceph-mon[23700]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:14.011 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:13 smithi154 ceph-mon[23700]: pgmap v250: 97 pgs: 97 active+clean; 141 MiB data, 389 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 648 op/s 2024-09-13T21:41:14.012 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:13 smithi154 ceph-mon[23700]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:14.012 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:13 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1176441518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:41:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:13 smithi181 ceph-mon[25811]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:13 smithi181 ceph-mon[25811]: pgmap v250: 97 pgs: 97 active+clean; 141 MiB data, 389 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 648 op/s 2024-09-13T21:41:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:13 smithi181 ceph-mon[25811]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:13 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1176441518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:41:14.357 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:41:14.357 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T21:41:14.357 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:41:14.357 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [], 2024-09-13T21:41:14.357 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "", 2024-09-13T21:41:14.358 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 image" 2024-09-13T21:41:14.358 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:41:14.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 11201 14.51 MB/sec warmup 27 sec latency 58.829 ms 2024-09-13T21:41:14.896 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:41:15.017 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:14 smithi154 ceph-mon[23700]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:15.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:14 smithi181 ceph-mon[25811]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:15.834 INFO:teuthology.orchestra.run.smithi154.stdout: 5 11502 14.10 MB/sec warmup 28 sec latency 64.215 ms 2024-09-13T21:41:16.045 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:15 smithi154 ceph-mon[23700]: pgmap v251: 97 pgs: 97 active+clean; 147 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 638 op/s 2024-09-13T21:41:16.045 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:15 smithi154 ceph-mon[23700]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:16.045 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:15 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1537334733' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:41:16.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:15 smithi181 ceph-mon[25811]: pgmap v251: 97 pgs: 97 active+clean; 147 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 638 op/s 2024-09-13T21:41:16.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:15 smithi181 ceph-mon[25811]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:16.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:15 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1537334733' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:41:16.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 11758 13.85 MB/sec warmup 29 sec latency 57.741 ms 2024-09-13T21:41:17.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 12223 13.91 MB/sec warmup 30 sec latency 37.339 ms 2024-09-13T21:41:18.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:17 smithi154 ceph-mon[23700]: pgmap v252: 97 pgs: 97 active+clean; 151 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 619 op/s 2024-09-13T21:41:18.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:17 smithi181 ceph-mon[25811]: pgmap v252: 97 pgs: 97 active+clean; 151 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 619 op/s 2024-09-13T21:41:18.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 12779 13.56 MB/sec warmup 31 sec latency 61.124 ms 2024-09-13T21:41:19.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 13025 13.33 MB/sec warmup 32 sec latency 98.046 ms 2024-09-13T21:41:20.141 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:19 smithi154 ceph-mon[23700]: pgmap v253: 97 pgs: 97 active+clean; 154 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.6 MiB/s wr, 412 op/s 2024-09-13T21:41:20.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:19 smithi181 ceph-mon[25811]: pgmap v253: 97 pgs: 97 active+clean; 154 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.6 MiB/s wr, 412 op/s 2024-09-13T21:41:20.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 13366 13.29 MB/sec warmup 33 sec latency 35.671 ms 2024-09-13T21:41:21.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 13930 13.56 MB/sec warmup 34 sec latency 66.472 ms 2024-09-13T21:41:22.160 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:21 smithi154 ceph-mon[23700]: pgmap v254: 97 pgs: 97 active+clean; 150 MiB data, 465 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-09-13T21:41:22.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:21 smithi181 ceph-mon[25811]: pgmap v254: 97 pgs: 97 active+clean; 150 MiB data, 465 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-09-13T21:41:22.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 14394 13.79 MB/sec warmup 35 sec latency 71.761 ms 2024-09-13T21:41:23.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 14873 13.56 MB/sec warmup 36 sec latency 43.894 ms 2024-09-13T21:41:23.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:23 smithi154 ceph-mon[23700]: pgmap v255: 97 pgs: 97 active+clean; 151 MiB data, 465 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 583 op/s 2024-09-13T21:41:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:23 smithi181 ceph-mon[25811]: pgmap v255: 97 pgs: 97 active+clean; 151 MiB data, 465 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 583 op/s 2024-09-13T21:41:24.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 15213 13.34 MB/sec warmup 37 sec latency 73.857 ms 2024-09-13T21:41:25.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 15505 13.25 MB/sec warmup 38 sec latency 130.144 ms 2024-09-13T21:41:26.012 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:25 smithi154 ceph-mon[23700]: pgmap v256: 97 pgs: 97 active+clean; 161 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.6 MiB/s wr, 578 op/s 2024-09-13T21:41:26.399 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:25 smithi181 ceph-mon[25811]: pgmap v256: 97 pgs: 97 active+clean; 161 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.6 MiB/s wr, 578 op/s 2024-09-13T21:41:26.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 15909 13.17 MB/sec warmup 39 sec latency 60.707 ms 2024-09-13T21:41:27.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 16379 12.94 MB/sec warmup 40 sec latency 39.038 ms 2024-09-13T21:41:28.041 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:27 smithi154 ceph-mon[23700]: pgmap v257: 97 pgs: 97 active+clean; 171 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 559 op/s 2024-09-13T21:41:28.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:27 smithi181 ceph-mon[25811]: pgmap v257: 97 pgs: 97 active+clean; 171 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 559 op/s 2024-09-13T21:41:28.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 16697 12.80 MB/sec warmup 41 sec latency 42.865 ms 2024-09-13T21:41:29.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 17269 13.15 MB/sec warmup 42 sec latency 38.085 ms 2024-09-13T21:41:30.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:29 smithi154 ceph-mon[23700]: pgmap v258: 97 pgs: 97 active+clean; 170 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 537 op/s 2024-09-13T21:41:30.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:29 smithi181 ceph-mon[25811]: pgmap v258: 97 pgs: 97 active+clean; 170 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 537 op/s 2024-09-13T21:41:30.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 17750 13.30 MB/sec warmup 43 sec latency 61.629 ms 2024-09-13T21:41:31.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 18038 13.22 MB/sec warmup 44 sec latency 51.360 ms 2024-09-13T21:41:32.161 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:31 smithi154 ceph-mon[23700]: pgmap v259: 97 pgs: 97 active+clean; 164 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.7 MiB/s wr, 589 op/s 2024-09-13T21:41:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:31 smithi181 ceph-mon[25811]: pgmap v259: 97 pgs: 97 active+clean; 164 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.7 MiB/s wr, 589 op/s 2024-09-13T21:41:32.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 18266 12.96 MB/sec warmup 45 sec latency 47.342 ms 2024-09-13T21:41:33.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 18655 12.77 MB/sec warmup 46 sec latency 86.717 ms 2024-09-13T21:41:33.969 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:33 smithi154 ceph-mon[23700]: pgmap v260: 97 pgs: 97 active+clean; 168 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.9 MiB/s wr, 574 op/s 2024-09-13T21:41:34.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:33 smithi181 ceph-mon[25811]: pgmap v260: 97 pgs: 97 active+clean; 168 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.9 MiB/s wr, 574 op/s 2024-09-13T21:41:34.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 19252 12.94 MB/sec warmup 47 sec latency 93.960 ms 2024-09-13T21:41:35.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 19907 12.76 MB/sec warmup 48 sec latency 30.012 ms 2024-09-13T21:41:36.037 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:35 smithi154 ceph-mon[23700]: pgmap v261: 97 pgs: 97 active+clean; 182 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.6 MiB/s wr, 626 op/s 2024-09-13T21:41:36.201 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:35 smithi181 ceph-mon[25811]: pgmap v261: 97 pgs: 97 active+clean; 182 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.6 MiB/s wr, 626 op/s 2024-09-13T21:41:36.835 INFO:teuthology.orchestra.run.smithi154.stdout: 5 20242 12.65 MB/sec warmup 49 sec latency 64.738 ms 2024-09-13T21:41:37.840 INFO:teuthology.orchestra.run.smithi154.stdout: 5 20718 12.86 MB/sec warmup 50 sec latency 67.129 ms 2024-09-13T21:41:38.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:37 smithi154 ceph-mon[23700]: pgmap v262: 97 pgs: 97 active+clean; 183 MiB data, 664 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 514 op/s 2024-09-13T21:41:38.399 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:37 smithi181 ceph-mon[25811]: pgmap v262: 97 pgs: 97 active+clean; 183 MiB data, 664 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 514 op/s 2024-09-13T21:41:38.841 INFO:teuthology.orchestra.run.smithi154.stdout: 5 21249 12.98 MB/sec warmup 51 sec latency 58.617 ms 2024-09-13T21:41:39.841 INFO:teuthology.orchestra.run.smithi154.stdout: 5 21640 13.03 MB/sec warmup 52 sec latency 49.764 ms 2024-09-13T21:41:40.144 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:39 smithi154 ceph-mon[23700]: pgmap v263: 97 pgs: 97 active+clean; 182 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 518 op/s 2024-09-13T21:41:40.286 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:39 smithi181 ceph-mon[25811]: pgmap v263: 97 pgs: 97 active+clean; 182 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 518 op/s 2024-09-13T21:41:40.841 INFO:teuthology.orchestra.run.smithi154.stdout: 5 21950 12.82 MB/sec warmup 53 sec latency 50.572 ms 2024-09-13T21:41:41.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:41 smithi154 ceph-mon[23700]: pgmap v264: 97 pgs: 97 active+clean; 186 MiB data, 705 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.5 MiB/s wr, 653 op/s 2024-09-13T21:41:41.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:41 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:41.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:41 smithi154 ceph-mon[23700]: Upgrade: Target is version 19.1.1-324-gecab7be5 (unknown) 2024-09-13T21:41:41.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:41 smithi154 ceph-mon[23700]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T21:41:41.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:41 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:41:41.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:41 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:41.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:41 smithi154 ceph-mon[23700]: Upgrade: Need to upgrade myself (mgr.smithi154.ayrwdi) 2024-09-13T21:41:41.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:41 smithi154 ceph-mon[23700]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 on smithi181 2024-09-13T21:41:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:41 smithi181 ceph-mon[25811]: pgmap v264: 97 pgs: 97 active+clean; 186 MiB data, 705 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.5 MiB/s wr, 653 op/s 2024-09-13T21:41:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:41 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:41 smithi181 ceph-mon[25811]: Upgrade: Target is version 19.1.1-324-gecab7be5 (unknown) 2024-09-13T21:41:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:41 smithi181 ceph-mon[25811]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T21:41:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:41 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:41:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:41 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:41:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:41 smithi181 ceph-mon[25811]: Upgrade: Need to upgrade myself (mgr.smithi154.ayrwdi) 2024-09-13T21:41:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:41 smithi181 ceph-mon[25811]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 on smithi181 2024-09-13T21:41:41.841 INFO:teuthology.orchestra.run.smithi154.stdout: 5 22280 12.65 MB/sec warmup 54 sec latency 50.737 ms 2024-09-13T21:41:42.841 INFO:teuthology.orchestra.run.smithi154.stdout: 5 22570 12.58 MB/sec warmup 55 sec latency 99.343 ms 2024-09-13T21:41:43.502 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:43 smithi154 ceph-mon[23700]: pgmap v265: 97 pgs: 97 active+clean; 192 MiB data, 736 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.7 MiB/s wr, 626 op/s 2024-09-13T21:41:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:43 smithi181 ceph-mon[25811]: pgmap v265: 97 pgs: 97 active+clean; 192 MiB data, 736 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.7 MiB/s wr, 626 op/s 2024-09-13T21:41:43.841 INFO:teuthology.orchestra.run.smithi154.stdout: 5 23054 12.59 MB/sec warmup 56 sec latency 59.002 ms 2024-09-13T21:41:44.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 23655 12.48 MB/sec warmup 57 sec latency 49.516 ms 2024-09-13T21:41:45.300 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:41:45.320 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:45 smithi154 ceph-mon[23700]: pgmap v266: 97 pgs: 97 active+clean; 192 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.6 MiB/s wr, 609 op/s 2024-09-13T21:41:45.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:45 smithi181 ceph-mon[25811]: pgmap v266: 97 pgs: 97 active+clean; 192 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.6 MiB/s wr, 609 op/s 2024-09-13T21:41:45.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 23907 12.43 MB/sec warmup 58 sec latency 48.069 ms 2024-09-13T21:41:46.038 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:41:46.038 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (6m) 60s ago 7m 22.1M - ba2b418f427c 4c0bd657c209 2024-09-13T21:41:46.038 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (7m) 60s ago 7m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:41:46.038 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (6m) 62s ago 6m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (5m) 60s ago 7m 52.4M - 8.3.5 dad864ee21e9 b275b298ebe6 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (92s) 60s ago 92s 12.6M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (90s) 62s ago 89s 14.7M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:9283 running (8m) 60s ago 8m 460M - 17.2.0 e1d6a67b021e c729166e328e 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283 running (6m) 62s ago 6m 413M - 17.2.0 e1d6a67b021e d2a641126ac6 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (8m) 60s ago 8m 48.4M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (6m) 62s ago 6m 37.9M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (71s) 60s ago 70s 59.9M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (65s) 62s ago 65s 46.7M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (7m) 60s ago 7m 17.4M - 1dbe0e931976 8ff8ec26e7ae 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (6m) 62s ago 6m 16.7M - 1dbe0e931976 31f26a6812e0 2024-09-13T21:41:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (5m) 60s ago 5m 43.2M 3216M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:41:46.040 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (5m) 60s ago 5m 43.5M 3216M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:41:46.040 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (5m) 60s ago 5m 40.6M 3216M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:41:46.040 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (4m) 60s ago 4m 42.0M 3216M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:41:46.040 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (4m) 62s ago 4m 39.0M 3983M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:41:46.040 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (3m) 62s ago 3m 39.7M 3983M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:41:46.040 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (3m) 62s ago 3m 39.3M 3983M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:41:46.040 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (3m) 62s ago 3m 41.2M 3983M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:41:46.040 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (6m) 60s ago 6m 63.2M - 514e6a882f6e ca18cf24d925 2024-09-13T21:41:46.347 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:46 smithi154 ceph-mon[23700]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:46.347 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:46 smithi154 ceph-mon[23700]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:46.347 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:41:46.347 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:46 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:41:46.516 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:41:46.516 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:41:46.517 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:41:46.517 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:41:46.517 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:41:46.517 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:41:46.517 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:41:46.517 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:41:46.518 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:41:46.518 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:41:46.518 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:41:46.518 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:41:46.518 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:41:46.518 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:41:46.519 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T21:41:46.519 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:41:46.519 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:41:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:46 smithi181 ceph-mon[25811]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:46 smithi181 ceph-mon[25811]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:41:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:46 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:41:46.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 24507 12.66 MB/sec warmup 59 sec latency 33.771 ms 2024-09-13T21:41:46.992 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:41:46.992 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:41:46.992 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:41:46.992 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [], 2024-09-13T21:41:46.992 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "0/23 daemons upgraded", 2024-09-13T21:41:46.992 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 image on host smithi181" 2024-09-13T21:41:46.992 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:41:47.351 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:47 smithi154 ceph-mon[23700]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:47.351 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:47 smithi154 ceph-mon[23700]: pgmap v267: 97 pgs: 97 active+clean; 192 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.2 MiB/s wr, 587 op/s 2024-09-13T21:41:47.351 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:47 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1529480512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:41:47.352 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:47 smithi154 ceph-mon[23700]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:47.559 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:41:47.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:47 smithi181 ceph-mon[25811]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:47.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:47 smithi181 ceph-mon[25811]: pgmap v267: 97 pgs: 97 active+clean; 192 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.2 MiB/s wr, 587 op/s 2024-09-13T21:41:47.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:47 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1529480512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:41:47.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:47 smithi181 ceph-mon[25811]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:41:47.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 25062 12.84 MB/sec warmup 60 sec latency 57.079 ms 2024-09-13T21:41:48.374 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:48 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/691868069' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:41:48.530 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:48 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/691868069' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:41:48.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 25475 12.70 MB/sec warmup 61 sec latency 64.323 ms 2024-09-13T21:41:49.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:49 smithi154 ceph-mon[23700]: pgmap v268: 97 pgs: 97 active+clean; 191 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 583 op/s 2024-09-13T21:41:49.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:49 smithi181 ceph-mon[25811]: pgmap v268: 97 pgs: 97 active+clean; 191 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 583 op/s 2024-09-13T21:41:49.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 26054 12.68 MB/sec warmup 62 sec latency 32.420 ms 2024-09-13T21:41:50.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 26817 12.73 MB/sec warmup 63 sec latency 54.065 ms 2024-09-13T21:41:51.413 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:51 smithi154 ceph-mon[23700]: pgmap v269: 97 pgs: 97 active+clean; 205 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.0 MiB/s wr, 702 op/s 2024-09-13T21:41:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:51 smithi181 ceph-mon[25811]: pgmap v269: 97 pgs: 97 active+clean; 205 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.0 MiB/s wr, 702 op/s 2024-09-13T21:41:51.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 27429 12.75 MB/sec warmup 64 sec latency 64.947 ms 2024-09-13T21:41:52.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 28060 12.98 MB/sec warmup 65 sec latency 72.108 ms 2024-09-13T21:41:53.438 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:53 smithi154 ceph-mon[23700]: pgmap v270: 97 pgs: 97 active+clean; 210 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.7 MiB/s wr, 668 op/s 2024-09-13T21:41:53.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:53 smithi181 ceph-mon[25811]: pgmap v270: 97 pgs: 97 active+clean; 210 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.7 MiB/s wr, 668 op/s 2024-09-13T21:41:53.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 28502 13.05 MB/sec warmup 66 sec latency 51.559 ms 2024-09-13T21:41:54.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 28807 12.97 MB/sec warmup 67 sec latency 55.458 ms 2024-09-13T21:41:55.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:55 smithi154 ceph-mon[23700]: pgmap v271: 97 pgs: 97 active+clean; 211 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 645 op/s 2024-09-13T21:41:55.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:55 smithi181 ceph-mon[25811]: pgmap v271: 97 pgs: 97 active+clean; 211 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 645 op/s 2024-09-13T21:41:55.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 29197 12.84 MB/sec warmup 68 sec latency 84.181 ms 2024-09-13T21:41:56.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 29775 12.83 MB/sec warmup 69 sec latency 41.541 ms 2024-09-13T21:41:57.476 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:57 smithi154 ceph-mon[23700]: pgmap v272: 97 pgs: 97 active+clean; 214 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.7 MiB/s wr, 724 op/s 2024-09-13T21:41:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:57 smithi181 ceph-mon[25811]: pgmap v272: 97 pgs: 97 active+clean; 214 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.7 MiB/s wr, 724 op/s 2024-09-13T21:41:57.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 30403 12.85 MB/sec warmup 70 sec latency 85.662 ms 2024-09-13T21:41:58.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 30890 12.81 MB/sec warmup 71 sec latency 52.733 ms 2024-09-13T21:41:59.500 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:41:59 smithi154 ceph-mon[23700]: pgmap v273: 97 pgs: 97 active+clean; 217 MiB data, 940 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.3 MiB/s wr, 669 op/s 2024-09-13T21:41:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:41:59 smithi181 ceph-mon[25811]: pgmap v273: 97 pgs: 97 active+clean; 217 MiB data, 940 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.3 MiB/s wr, 669 op/s 2024-09-13T21:41:59.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 31202 12.77 MB/sec warmup 72 sec latency 61.882 ms 2024-09-13T21:42:00.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 31661 12.91 MB/sec warmup 73 sec latency 74.861 ms 2024-09-13T21:42:01.365 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:01 smithi154 ceph-mon[23700]: pgmap v274: 97 pgs: 97 active+clean; 220 MiB data, 958 MiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.2 MiB/s wr, 829 op/s 2024-09-13T21:42:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:01 smithi181 ceph-mon[25811]: pgmap v274: 97 pgs: 97 active+clean; 220 MiB data, 958 MiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.2 MiB/s wr, 829 op/s 2024-09-13T21:42:01.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 32061 12.93 MB/sec warmup 74 sec latency 94.516 ms 2024-09-13T21:42:02.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 32390 12.85 MB/sec warmup 75 sec latency 59.717 ms 2024-09-13T21:42:03.353 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:03 smithi154 ceph-mon[23700]: pgmap v275: 97 pgs: 97 active+clean; 218 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 657 op/s 2024-09-13T21:42:03.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:03 smithi181 ceph-mon[25811]: pgmap v275: 97 pgs: 97 active+clean; 218 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 657 op/s 2024-09-13T21:42:03.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 32779 12.76 MB/sec warmup 76 sec latency 46.444 ms 2024-09-13T21:42:04.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 33231 12.72 MB/sec warmup 77 sec latency 70.420 ms 2024-09-13T21:42:05.367 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:05 smithi154 ceph-mon[23700]: pgmap v276: 97 pgs: 97 active+clean; 228 MiB data, 1005 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 685 op/s 2024-09-13T21:42:05.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:05 smithi181 ceph-mon[25811]: pgmap v276: 97 pgs: 97 active+clean; 228 MiB data, 1005 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 685 op/s 2024-09-13T21:42:05.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 33937 12.77 MB/sec warmup 78 sec latency 30.015 ms 2024-09-13T21:42:06.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 34782 12.86 MB/sec warmup 79 sec latency 22.190 ms 2024-09-13T21:42:07.380 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:07 smithi154 ceph-mon[23700]: pgmap v277: 97 pgs: 97 active+clean; 231 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.4 MiB/s wr, 665 op/s 2024-09-13T21:42:07.457 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:07 smithi181 ceph-mon[25811]: pgmap v277: 97 pgs: 97 active+clean; 231 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.4 MiB/s wr, 665 op/s 2024-09-13T21:42:07.851 INFO:teuthology.orchestra.run.smithi154.stdout: 5 35633 13.16 MB/sec warmup 80 sec latency 16.866 ms 2024-09-13T21:42:08.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 36283 13.13 MB/sec warmup 81 sec latency 17.208 ms 2024-09-13T21:42:09.391 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:09 smithi154 ceph-mon[23700]: pgmap v278: 97 pgs: 97 active+clean; 244 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 757 op/s 2024-09-13T21:42:09.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:09 smithi154 ceph-mon[23700]: Upgrade: Updating mgr.smithi181.byvkib 2024-09-13T21:42:09.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:09 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:09.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:09 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:42:09.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:09 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:42:09.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:09 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:42:09.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:09 smithi154 ceph-mon[23700]: Deploying daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:42:09.569 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:09 smithi181 ceph-mon[25811]: pgmap v278: 97 pgs: 97 active+clean; 244 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 757 op/s 2024-09-13T21:42:09.569 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:09 smithi181 ceph-mon[25811]: Upgrade: Updating mgr.smithi181.byvkib 2024-09-13T21:42:09.569 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:09 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:09.569 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:09 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:42:09.569 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:09 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:42:09.569 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:09 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:42:09.570 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:09 smithi181 ceph-mon[25811]: Deploying daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:42:09.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 37066 13.19 MB/sec warmup 82 sec latency 20.532 ms 2024-09-13T21:42:10.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 37863 13.24 MB/sec warmup 83 sec latency 25.871 ms 2024-09-13T21:42:11.402 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:11 smithi154 ceph-mon[23700]: pgmap v279: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.6 MiB/s wr, 763 op/s 2024-09-13T21:42:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:11 smithi181 ceph-mon[25811]: pgmap v279: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.6 MiB/s wr, 763 op/s 2024-09-13T21:42:11.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 38616 13.41 MB/sec warmup 84 sec latency 22.067 ms 2024-09-13T21:42:12.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 39451 13.59 MB/sec warmup 85 sec latency 21.879 ms 2024-09-13T21:42:13.413 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:13 smithi154 ceph-mon[23700]: pgmap v280: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.6 MiB/s wr, 647 op/s 2024-09-13T21:42:13.513 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:13 smithi181 ceph-mon[25811]: pgmap v280: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.6 MiB/s wr, 647 op/s 2024-09-13T21:42:13.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 40325 13.62 MB/sec warmup 86 sec latency 12.380 ms 2024-09-13T21:42:14.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 41221 13.69 MB/sec warmup 87 sec latency 52.568 ms 2024-09-13T21:42:15.433 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:15 smithi154 ceph-mon[23700]: pgmap v281: 97 pgs: 97 active+clean; 254 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 931 op/s 2024-09-13T21:42:15.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:15 smithi181 ceph-mon[25811]: pgmap v281: 97 pgs: 97 active+clean; 254 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.3 MiB/s wr, 931 op/s 2024-09-13T21:42:15.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 41959 13.79 MB/sec warmup 88 sec latency 110.184 ms 2024-09-13T21:42:16.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 42715 13.98 MB/sec warmup 89 sec latency 82.567 ms 2024-09-13T21:42:17.487 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:17 smithi154 ceph-mon[23700]: pgmap v282: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 923 op/s 2024-09-13T21:42:17.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:17 smithi181 ceph-mon[25811]: pgmap v282: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 923 op/s 2024-09-13T21:42:17.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 43375 13.95 MB/sec warmup 90 sec latency 19.719 ms 2024-09-13T21:42:18.105 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:42:18.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 44015 13.94 MB/sec warmup 91 sec latency 33.447 ms 2024-09-13T21:42:19.084 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:42:19.084 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (6m) 93s ago 7m 22.1M - ba2b418f427c 4c0bd657c209 2024-09-13T21:42:19.084 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (7m) 93s ago 7m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (7m) 95s ago 7m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (6m) 93s ago 7m 52.4M - 8.3.5 dad864ee21e9 b275b298ebe6 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (2m) 93s ago 2m 12.6M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (2m) 95s ago 2m 14.7M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:9283 running (9m) 93s ago 9m 460M - 17.2.0 e1d6a67b021e c729166e328e 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283 starting - - - - 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (9m) 93s ago 9m 48.4M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (7m) 95s ago 7m 37.9M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (104s) 93s ago 103s 59.9M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (98s) 95s ago 98s 46.7M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (7m) 93s ago 7m 17.4M - 1dbe0e931976 8ff8ec26e7ae 2024-09-13T21:42:19.085 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (7m) 95s ago 7m 16.7M - 1dbe0e931976 31f26a6812e0 2024-09-13T21:42:19.086 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (6m) 93s ago 6m 43.2M 3216M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:42:19.086 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (6m) 93s ago 6m 43.5M 3216M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:42:19.086 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (5m) 93s ago 5m 40.6M 3216M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:42:19.086 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (5m) 93s ago 5m 42.0M 3216M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:42:19.086 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (4m) 95s ago 4m 39.0M 3983M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:42:19.086 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (4m) 95s ago 4m 39.7M 3983M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:42:19.086 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (3m) 95s ago 3m 39.3M 3983M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:42:19.087 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (3m) 95s ago 3m 41.2M 3983M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:42:19.087 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (6m) 93s ago 6m 63.2M - 514e6a882f6e ca18cf24d925 2024-09-13T21:42:19.455 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:19 smithi154 ceph-mon[23700]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:19.455 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:19 smithi154 ceph-mon[23700]: pgmap v283: 97 pgs: 97 active+clean; 267 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-09-13T21:42:19.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:19.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:42:19.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:42:19.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:19 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:42:19.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:19 smithi154 ceph-mon[23700]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:19.589 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:42:19.589 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:42:19.589 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:42:19.589 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:42:19.589 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:42:19.589 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:42:19.589 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:42:19.590 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:42:19.590 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:19 smithi181 ceph-mon[25811]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:19.591 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:19 smithi181 ceph-mon[25811]: pgmap v283: 97 pgs: 97 active+clean; 267 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-09-13T21:42:19.591 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:19.591 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:42:19.591 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:42:19.591 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:19 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:42:19.591 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:19 smithi181 ceph-mon[25811]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:19.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 44646 13.95 MB/sec warmup 92 sec latency 44.149 ms 2024-09-13T21:42:20.025 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:42:20.026 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:42:20.026 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:42:20.026 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [], 2024-09-13T21:42:20.026 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "", 2024-09-13T21:42:20.026 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading mgr daemons" 2024-09-13T21:42:20.026 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:42:20.460 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:20 smithi154 ceph-mon[23700]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:20.460 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:20 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/2192506062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:42:20.606 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:42:20.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:20 smithi181 ceph-mon[25811]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:20.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:20 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2192506062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:42:20.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 45245 13.96 MB/sec warmup 93 sec latency 51.697 ms 2024-09-13T21:42:21.481 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:21 smithi154 ceph-mon[23700]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:21.482 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:21 smithi154 ceph-mon[23700]: pgmap v284: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 962 op/s 2024-09-13T21:42:21.482 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:21 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1021829542' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:42:21.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:21 smithi181 ceph-mon[25811]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:21.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:21 smithi181 ceph-mon[25811]: pgmap v284: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 962 op/s 2024-09-13T21:42:21.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:21 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1021829542' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:42:21.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 45505 13.92 MB/sec warmup 94 sec latency 62.676 ms 2024-09-13T21:42:22.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 46157 14.04 MB/sec warmup 95 sec latency 105.913 ms 2024-09-13T21:42:23.250 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:23.250 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:23 smithi154 ceph-mon[23700]: pgmap v285: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 934 op/s 2024-09-13T21:42:23.250 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:23 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:23 smithi181 ceph-mon[25811]: pgmap v285: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 934 op/s 2024-09-13T21:42:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:23 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:23.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 46684 14.06 MB/sec warmup 96 sec latency 51.439 ms 2024-09-13T21:42:24.254 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:24 smithi154 ceph-mon[23700]: Standby manager daemon smithi181.byvkib restarted 2024-09-13T21:42:24.255 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:24 smithi154 ceph-mon[23700]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:42:24.255 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:24 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:42:24.255 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:24 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:42:24.255 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:24 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:42:24.255 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:24 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:42:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:24 smithi181 ceph-mon[25811]: Standby manager daemon smithi181.byvkib restarted 2024-09-13T21:42:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:24 smithi181 ceph-mon[25811]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:42:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:24 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:42:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:24 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:42:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:24 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:42:24.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:24 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:42:24.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 47428 14.01 MB/sec warmup 97 sec latency 12.917 ms 2024-09-13T21:42:25.260 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:25 smithi154 ceph-mon[23700]: pgmap v286: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-09-13T21:42:25.260 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:25 smithi154 ceph-mon[23700]: mgrmap e22: smithi154.ayrwdi(active, since 7m), standbys: smithi181.byvkib 2024-09-13T21:42:25.340 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:25 smithi181 ceph-mon[25811]: pgmap v286: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-09-13T21:42:25.340 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:25 smithi181 ceph-mon[25811]: mgrmap e22: smithi154.ayrwdi(active, since 7m), standbys: smithi181.byvkib 2024-09-13T21:42:25.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 48117 14.05 MB/sec warmup 98 sec latency 28.507 ms 2024-09-13T21:42:26.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 48460 13.98 MB/sec warmup 99 sec latency 78.192 ms 2024-09-13T21:42:27.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 48870 13.96 MB/sec warmup 100 sec latency 66.168 ms 2024-09-13T21:42:28.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: pgmap v287: 97 pgs: 97 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.9 MiB/s wr, 800 op/s 2024-09-13T21:42:28.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:28.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:28.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:28.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:42:28.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: Upgrade: Need to upgrade myself (mgr.smithi154.ayrwdi) 2024-09-13T21:42:28.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: Failing over to other MGR 2024-09-13T21:42:28.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr fail", "who": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:42:28.112 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:27 smithi154 ceph-mon[23700]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: pgmap v287: 97 pgs: 97 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.9 MiB/s wr, 800 op/s 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: Upgrade: Need to upgrade myself (mgr.smithi154.ayrwdi) 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: Failing over to other MGR 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr fail", "who": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:42:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:27 smithi181 ceph-mon[25811]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T21:42:28.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 49433 14.06 MB/sec warmup 101 sec latency 44.757 ms 2024-09-13T21:42:28.888 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "mgr fail", "who": "smithi154.ayrwdi"}]': finished 2024-09-13T21:42:28.888 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: mgrmap e23: smithi181.byvkib(active, starting, since 0.990859s) 2024-09-13T21:42:28.888 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:42:28.888 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:42:28.888 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:42:28.888 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr metadata", "who": "smithi181.byvkib", "id": "smithi181.byvkib"}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: Manager daemon smithi181.byvkib is now available 2024-09-13T21:42:28.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:28.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: Checking for cert/key for grafana.smithi154 2024-09-13T21:42:28.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:28.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:42:28.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:42:28.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.byvkib/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:42:28.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.byvkib/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:42:28.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.byvkib/trash_purge_schedule"}]: dispatch 2024-09-13T21:42:28.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:28 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.byvkib/trash_purge_schedule"}]: dispatch 2024-09-13T21:42:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.14202 172.21.15.154:0/971459148' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "mgr fail", "who": "smithi154.ayrwdi"}]': finished 2024-09-13T21:42:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: mgrmap e23: smithi181.byvkib(active, starting, since 0.990859s) 2024-09-13T21:42:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:42:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:42:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:42:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:42:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr metadata", "who": "smithi181.byvkib", "id": "smithi181.byvkib"}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: Manager daemon smithi181.byvkib is now available 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:29.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: Checking for cert/key for grafana.smithi154 2024-09-13T21:42:29.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:29.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:42:29.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:42:29.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.byvkib/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:42:29.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.byvkib/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:42:29.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.byvkib/trash_purge_schedule"}]: dispatch 2024-09-13T21:42:29.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:28 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.byvkib/trash_purge_schedule"}]: dispatch 2024-09-13T21:42:29.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 49954 14.11 MB/sec warmup 102 sec latency 32.642 ms 2024-09-13T21:42:29.904 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:29 smithi154 ceph-mon[23700]: Deploying cephadm binary to smithi181 2024-09-13T21:42:29.905 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:29 smithi154 ceph-mon[23700]: mgrmap e24: smithi181.byvkib(active, since 1.99893s) 2024-09-13T21:42:29.905 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:29 smithi154 ceph-mon[23700]: pgmap v3: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T21:42:29.905 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:29 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:29.905 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:29 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:29 smithi181 ceph-mon[25811]: Deploying cephadm binary to smithi181 2024-09-13T21:42:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:29 smithi181 ceph-mon[25811]: mgrmap e24: smithi181.byvkib(active, since 1.99893s) 2024-09-13T21:42:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:29 smithi181 ceph-mon[25811]: pgmap v3: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T21:42:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:29 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:29 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:30.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 50233 14.03 MB/sec warmup 103 sec latency 38.304 ms 2024-09-13T21:42:31.420 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:31 smithi154 ceph-mon[23700]: Deploying cephadm binary to smithi154 2024-09-13T21:42:31.420 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:31 smithi154 ceph-mon[23700]: pgmap v4: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T21:42:31.421 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:31 smithi154 ceph-mon[23700]: [13/Sep/2024:21:42:30] ENGINE Bus STARTING 2024-09-13T21:42:31.497 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:31 smithi181 ceph-mon[25811]: Deploying cephadm binary to smithi154 2024-09-13T21:42:31.497 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:31 smithi181 ceph-mon[25811]: pgmap v4: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T21:42:31.497 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:31 smithi181 ceph-mon[25811]: [13/Sep/2024:21:42:30] ENGINE Bus STARTING 2024-09-13T21:42:31.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 50660 13.94 MB/sec warmup 104 sec latency 46.948 ms 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:32 smithi181 ceph-mon[25811]: [13/Sep/2024:21:42:30] ENGINE Serving on http://172.21.15.181:8765 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:32 smithi181 ceph-mon[25811]: [13/Sep/2024:21:42:30] ENGINE Serving on https://172.21.15.181:7150 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:32 smithi181 ceph-mon[25811]: [13/Sep/2024:21:42:30] ENGINE Bus STARTED 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:32 smithi181 ceph-mon[25811]: [13/Sep/2024:21:42:30] ENGINE Error in HTTPServer.serve 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Traceback (most recent call last): 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._connections.run(self.expiration_interval) 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._run(expiration_interval) 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T21:42:32.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: s = self.context.wrap_socket( 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: return self.sslsocket_class._create( 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self.do_handshake() 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._sslobj.do_handshake() 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T21:42:32.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:32 smithi181 ceph-mon[25811]: mgrmap e25: smithi181.byvkib(active, since 4s) 2024-09-13T21:42:32.426 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:32 smithi154 ceph-mon[23700]: [13/Sep/2024:21:42:30] ENGINE Serving on http://172.21.15.181:8765 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:32 smithi154 ceph-mon[23700]: [13/Sep/2024:21:42:30] ENGINE Serving on https://172.21.15.181:7150 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:32 smithi154 ceph-mon[23700]: [13/Sep/2024:21:42:30] ENGINE Bus STARTED 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:32 smithi154 ceph-mon[23700]: [13/Sep/2024:21:42:30] ENGINE Error in HTTPServer.serve 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Traceback (most recent call last): 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._connections.run(self.expiration_interval) 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._run(expiration_interval) 2024-09-13T21:42:32.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T21:42:32.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T21:42:32.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T21:42:32.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T21:42:32.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T21:42:32.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: s = self.context.wrap_socket( 2024-09-13T21:42:32.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T21:42:32.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: return self.sslsocket_class._create( 2024-09-13T21:42:32.428 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T21:42:32.429 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self.do_handshake() 2024-09-13T21:42:32.429 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T21:42:32.429 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._sslobj.do_handshake() 2024-09-13T21:42:32.429 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T21:42:32.429 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:32 smithi154 ceph-mon[23700]: mgrmap e25: smithi181.byvkib(active, since 4s) 2024-09-13T21:42:32.852 INFO:teuthology.orchestra.run.smithi154.stdout: 5 51126 13.92 MB/sec warmup 105 sec latency 49.042 ms 2024-09-13T21:42:33.445 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:33 smithi154 ceph-mon[23700]: pgmap v5: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T21:42:33.445 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:33 smithi154 ceph-mon[23700]: mgrmap e26: smithi181.byvkib(active, since 5s) 2024-09-13T21:42:33.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:33 smithi181 ceph-mon[25811]: pgmap v5: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T21:42:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:33 smithi181 ceph-mon[25811]: mgrmap e26: smithi181.byvkib(active, since 5s) 2024-09-13T21:42:33.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 51654 13.92 MB/sec warmup 106 sec latency 44.192 ms 2024-09-13T21:42:34.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 52229 13.87 MB/sec warmup 107 sec latency 42.036 ms 2024-09-13T21:42:35.208 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:35 smithi154 ceph-mon[23700]: pgmap v6: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T21:42:35.208 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:35 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:35.209 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:35 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:35.209 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:35 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:35.209 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:35 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:35.496 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:35 smithi181 ceph-mon[25811]: pgmap v6: 97 pgs: 97 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T21:42:35.497 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:35 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:35.497 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:35 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:35.497 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:35 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:35.497 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:35 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:35.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 52586 13.90 MB/sec warmup 108 sec latency 53.437 ms 2024-09-13T21:42:36.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 53163 13.96 MB/sec warmup 109 sec latency 39.786 ms 2024-09-13T21:42:37.230 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:37 smithi154 ceph-mon[23700]: pgmap v7: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 4.3 MiB/s wr, 484 op/s 2024-09-13T21:42:37.230 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:37 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.154:0/886330312' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.ayrwdi/crt"}]: dispatch 2024-09-13T21:42:37.230 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:37 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.154:0/886330312' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:42:37.230 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:37 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.154:0/886330312' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.ayrwdi/key"}]: dispatch 2024-09-13T21:42:37.231 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:37 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.154:0/886330312' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:42:37.231 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:37 smithi154 ceph-mon[23700]: Standby manager daemon smithi154.ayrwdi started 2024-09-13T21:42:37.626 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:37 smithi181 ceph-mon[25811]: pgmap v7: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 4.3 MiB/s wr, 484 op/s 2024-09-13T21:42:37.626 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:37 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.154:0/886330312' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.ayrwdi/crt"}]: dispatch 2024-09-13T21:42:37.626 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:37 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.154:0/886330312' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:42:37.626 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:37 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.154:0/886330312' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.ayrwdi/key"}]: dispatch 2024-09-13T21:42:37.626 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:37 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.154:0/886330312' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:42:37.627 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:37 smithi181 ceph-mon[25811]: Standby manager daemon smithi154.ayrwdi started 2024-09-13T21:42:37.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 53623 13.98 MB/sec warmup 110 sec latency 35.324 ms 2024-09-13T21:42:38.240 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:38 smithi154 ceph-mon[23700]: mgrmap e27: smithi181.byvkib(active, since 10s), standbys: smithi154.ayrwdi 2024-09-13T21:42:38.240 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:38 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:42:38.475 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:38 smithi181 ceph-mon[25811]: mgrmap e27: smithi181.byvkib(active, since 10s), standbys: smithi154.ayrwdi 2024-09-13T21:42:38.475 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:38 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:42:38.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 53926 13.91 MB/sec warmup 111 sec latency 51.189 ms 2024-09-13T21:42:39.255 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: pgmap v8: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 3.3 MiB/s wr, 376 op/s 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:39 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: pgmap v8: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 3.3 MiB/s wr, 376 op/s 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:39.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:39 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:39.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 54447 13.83 MB/sec warmup 112 sec latency 68.732 ms 2024-09-13T21:42:40.278 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:40 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi181 to 2703M 2024-09-13T21:42:40.278 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:40 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:40.279 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:40 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:40.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:40 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi181 to 2703M 2024-09-13T21:42:40.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:40 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:40.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:40 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:40.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 55162 13.88 MB/sec warmup 113 sec latency 22.654 ms 2024-09-13T21:42:41.284 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: pgmap v9: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 6.2 MiB/s wr, 597 op/s 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:41.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:42:41.286 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:41 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:42:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: pgmap v9: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 6.2 MiB/s wr, 597 op/s 2024-09-13T21:42:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T21:42:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:41.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:42:41.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:41 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:42:41.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 55663 13.84 MB/sec warmup 114 sec latency 43.407 ms 2024-09-13T21:42:42.294 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: Adjusting osd_memory_target on smithi154 to 1936M 2024-09-13T21:42:42.294 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:42:42.294 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: Updating smithi181:/etc/ceph/ceph.conf 2024-09-13T21:42:42.295 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:42:42.295 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:42:42.295 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:42:42.295 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.295 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.295 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.296 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.296 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.296 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.296 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.296 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.296 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.296 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:42 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: Adjusting osd_memory_target on smithi154 to 1936M 2024-09-13T21:42:42.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:42:42.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: Updating smithi181:/etc/ceph/ceph.conf 2024-09-13T21:42:42.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:42:42.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:42:42.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:42:42.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:42 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:42.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 56093 13.85 MB/sec warmup 115 sec latency 71.089 ms 2024-09-13T21:42:43.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:43 smithi154 ceph-mon[23700]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:42:43.300 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:43 smithi154 ceph-mon[23700]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:42:43.300 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:43 smithi154 ceph-mon[23700]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:42:43.300 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:43 smithi154 ceph-mon[23700]: pgmap v10: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 5.6 MiB/s wr, 546 op/s 2024-09-13T21:42:43.300 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:43 smithi154 ceph-mon[23700]: Reconfiguring alertmanager.smithi154 (dependencies changed)... 2024-09-13T21:42:43.300 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:43 smithi154 ceph-mon[23700]: Deploying daemon alertmanager.smithi154 on smithi154 2024-09-13T21:42:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:43 smithi181 ceph-mon[25811]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:42:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:43 smithi181 ceph-mon[25811]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:42:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:43 smithi181 ceph-mon[25811]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:42:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:43 smithi181 ceph-mon[25811]: pgmap v10: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 5.6 MiB/s wr, 546 op/s 2024-09-13T21:42:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:43 smithi181 ceph-mon[25811]: Reconfiguring alertmanager.smithi154 (dependencies changed)... 2024-09-13T21:42:43.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:43 smithi181 ceph-mon[25811]: Deploying daemon alertmanager.smithi154 on smithi154 2024-09-13T21:42:43.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 56900 14.02 MB/sec warmup 116 sec latency 39.827 ms 2024-09-13T21:42:44.303 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:44 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:42:44.320 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:44 smithi154 ceph-mon[23700]: pgmap v11: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 5.6 MiB/s wr, 546 op/s 2024-09-13T21:42:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:44 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:42:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:44 smithi181 ceph-mon[25811]: pgmap v11: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 5.6 MiB/s wr, 546 op/s 2024-09-13T21:42:44.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 57485 14.02 MB/sec warmup 117 sec latency 22.870 ms 2024-09-13T21:42:45.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 58229 14.00 MB/sec warmup 118 sec latency 53.444 ms 2024-09-13T21:42:46.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 59014 14.04 MB/sec warmup 119 sec latency 25.245 ms 2024-09-13T21:42:47.315 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:47 smithi154 ceph-mon[23700]: pgmap v12: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 9.3 MiB/s wr, 891 op/s 2024-09-13T21:42:47.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:47 smithi181 ceph-mon[25811]: pgmap v12: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 9.3 MiB/s wr, 891 op/s 2024-09-13T21:42:48.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 60283 25.20 MB/sec execute 1 sec latency 29.945 ms 2024-09-13T21:42:49.345 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:49 smithi154 ceph-mon[23700]: pgmap v13: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.8 MiB/s wr, 610 op/s 2024-09-13T21:42:49.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:49 smithi181 ceph-mon[25811]: pgmap v13: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.8 MiB/s wr, 610 op/s 2024-09-13T21:42:49.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 60719 20.41 MB/sec execute 2 sec latency 42.165 ms 2024-09-13T21:42:50.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 61096 15.84 MB/sec execute 3 sec latency 129.971 ms 2024-09-13T21:42:51.226 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:42:51.380 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:51 smithi154 ceph-mon[23700]: pgmap v14: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 996 op/s 2024-09-13T21:42:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:51 smithi181 ceph-mon[25811]: pgmap v14: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 996 op/s 2024-09-13T21:42:51.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 61629 13.56 MB/sec execute 4 sec latency 33.724 ms 2024-09-13T21:42:52.142 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:42:52.142 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (7m) 12s ago 8m 25.7M - 0.23.0 ba2b418f427c 4c0bd657c209 2024-09-13T21:42:52.142 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (8m) 12s ago 8m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:42:52.142 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (7m) 17s ago 7m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:42:52.142 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (6m) 12s ago 8m 52.4M - 8.3.5 dad864ee21e9 b275b298ebe6 2024-09-13T21:42:52.142 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (2m) 12s ago 2m 34.0M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:42:52.142 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (2m) 17s ago 2m 15.4M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:42:52.142 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:9283 running (9m) 12s ago 9m 168M - 17.2.0 e1d6a67b021e c729166e328e 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283 running (35s) 17s ago 7m 478M - 19.1.1-324-gecab7be5 0652ff2b5d78 01d71e511aaf 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (9m) 12s ago 9m 52.7M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (7m) 17s ago 7m 40.0M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (2m) 12s ago 2m 253M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (2m) 17s ago 2m 71.4M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (8m) 12s ago 8m 18.5M - 1.3.1 1dbe0e931976 8ff8ec26e7ae 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (7m) 17s ago 7m 17.0M - 1.3.1 1dbe0e931976 31f26a6812e0 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (7m) 12s ago 7m 250M 1936M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (6m) 12s ago 6m 183M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (6m) 12s ago 6m 157M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (5m) 12s ago 5m 195M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (5m) 17s ago 5m 218M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:42:52.143 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (4m) 17s ago 4m 212M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:42:52.144 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (4m) 17s ago 4m 200M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:42:52.144 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (4m) 17s ago 4m 297M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:42:52.144 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (7m) 12s ago 7m 65.4M - 2.33.4 514e6a882f6e ca18cf24d925 2024-09-13T21:42:52.413 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:52 smithi154 ceph-mon[23700]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:52.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:52 smithi181 ceph-mon[25811]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:52.775 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:42:52.776 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:42:52.777 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:42:52.777 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:42:52.777 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:42:52.777 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T21:42:52.777 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T21:42:52.777 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:42:52.777 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:42:52.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 62205 14.87 MB/sec execute 5 sec latency 32.822 ms 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [], 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "", 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:42:53.341 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:42:53.421 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:53 smithi154 ceph-mon[23700]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:53.421 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:53 smithi154 ceph-mon[23700]: pgmap v15: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 731 op/s 2024-09-13T21:42:53.421 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:53 smithi154 ceph-mon[23700]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:53.421 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:53 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3832560347' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:42:53.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:53 smithi181 ceph-mon[25811]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:53.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:53 smithi181 ceph-mon[25811]: pgmap v15: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 731 op/s 2024-09-13T21:42:53.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:53 smithi181 ceph-mon[25811]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:53.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:53 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/3832560347' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:42:53.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 62680 13.35 MB/sec execute 6 sec latency 98.149 ms 2024-09-13T21:42:53.983 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:42:54.240 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:54 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/188398799' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:42:54.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:54 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/188398799' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:42:54.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 63090 13.23 MB/sec execute 7 sec latency 133.120 ms 2024-09-13T21:42:55.282 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:55 smithi154 ceph-mon[23700]: from='client.24469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:55.283 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:55 smithi154 ceph-mon[23700]: pgmap v16: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 731 op/s 2024-09-13T21:42:55.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:55 smithi181 ceph-mon[25811]: from='client.24469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:42:55.653 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:55 smithi181 ceph-mon[25811]: pgmap v16: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 731 op/s 2024-09-13T21:42:55.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 63427 13.35 MB/sec execute 8 sec latency 109.816 ms 2024-09-13T21:42:56.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 63687 12.90 MB/sec execute 9 sec latency 121.674 ms 2024-09-13T21:42:57.367 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:57 smithi154 ceph-mon[23700]: pgmap v17: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-09-13T21:42:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:57 smithi181 ceph-mon[25811]: pgmap v17: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-09-13T21:42:57.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 63979 12.56 MB/sec execute 10 sec latency 68.912 ms 2024-09-13T21:42:58.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 64279 12.49 MB/sec execute 11 sec latency 69.813 ms 2024-09-13T21:42:59.423 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:59 smithi154 ceph-mon[23700]: pgmap v18: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 721 op/s 2024-09-13T21:42:59.424 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:59 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:42:59.424 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:59 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:59.424 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:42:59 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:59 smithi181 ceph-mon[25811]: pgmap v18: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 721 op/s 2024-09-13T21:42:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:59 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:42:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:59 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:42:59 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:42:59.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 64558 11.89 MB/sec execute 12 sec latency 93.135 ms 2024-09-13T21:43:00.446 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:00 smithi154 ceph-mon[23700]: Reconfiguring grafana.smithi154 (dependencies changed)... 2024-09-13T21:43:00.446 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:00 smithi154 ceph-mon[23700]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T21:43:00.447 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:00 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:00.447 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:00 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:00.447 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:00 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:43:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:00 smithi181 ceph-mon[25811]: Reconfiguring grafana.smithi154 (dependencies changed)... 2024-09-13T21:43:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:00 smithi181 ceph-mon[25811]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T21:43:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:00 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:00 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:00 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:43:00.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 64983 11.31 MB/sec execute 13 sec latency 35.418 ms 2024-09-13T21:43:01.478 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:01 smithi154 ceph-mon[23700]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:43:01.479 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:01 smithi154 ceph-mon[23700]: Reconfiguring daemon grafana.smithi154 on smithi154 2024-09-13T21:43:01.479 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:01 smithi154 ceph-mon[23700]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 928 op/s 2024-09-13T21:43:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:01 smithi181 ceph-mon[25811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T21:43:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:01 smithi181 ceph-mon[25811]: Reconfiguring daemon grafana.smithi154 on smithi154 2024-09-13T21:43:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:01 smithi181 ceph-mon[25811]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 928 op/s 2024-09-13T21:43:01.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 65245 11.02 MB/sec execute 14 sec latency 94.234 ms 2024-09-13T21:43:02.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 65629 11.09 MB/sec execute 15 sec latency 56.009 ms 2024-09-13T21:43:03.258 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:03 smithi154 ceph-mon[23700]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 542 op/s 2024-09-13T21:43:03.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:03 smithi181 ceph-mon[25811]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 542 op/s 2024-09-13T21:43:03.853 INFO:teuthology.orchestra.run.smithi154.stdout: 5 66020 10.85 MB/sec execute 16 sec latency 73.883 ms 2024-09-13T21:43:04.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 66283 10.42 MB/sec execute 17 sec latency 71.880 ms 2024-09-13T21:43:05.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:05 smithi154 ceph-mon[23700]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 542 op/s 2024-09-13T21:43:05.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:05 smithi181 ceph-mon[25811]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 542 op/s 2024-09-13T21:43:05.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 66610 10.39 MB/sec execute 18 sec latency 49.166 ms 2024-09-13T21:43:06.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 66976 10.72 MB/sec execute 19 sec latency 79.850 ms 2024-09-13T21:43:07.392 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:07 smithi154 ceph-mon[23700]: pgmap v22: 97 pgs: 97 active+clean; 316 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 759 op/s 2024-09-13T21:43:07.393 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:07 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:07.393 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:07 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:07 smithi181 ceph-mon[25811]: pgmap v22: 97 pgs: 97 active+clean; 316 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 759 op/s 2024-09-13T21:43:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:07 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:07 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:07.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 67441 10.99 MB/sec execute 20 sec latency 72.250 ms 2024-09-13T21:43:08.446 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:08 smithi154 ceph-mon[23700]: Reconfiguring node-exporter.smithi154 (dependencies changed)... 2024-09-13T21:43:08.446 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:08 smithi154 ceph-mon[23700]: Deploying daemon node-exporter.smithi154 on smithi154 2024-09-13T21:43:08.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:08 smithi181 ceph-mon[25811]: Reconfiguring node-exporter.smithi154 (dependencies changed)... 2024-09-13T21:43:08.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:08 smithi181 ceph-mon[25811]: Deploying daemon node-exporter.smithi154 on smithi154 2024-09-13T21:43:08.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 67735 10.91 MB/sec execute 21 sec latency 59.346 ms 2024-09-13T21:43:09.486 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:09 smithi154 ceph-mon[23700]: pgmap v23: 97 pgs: 97 active+clean; 316 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 424 op/s 2024-09-13T21:43:09.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:09 smithi181 ceph-mon[25811]: pgmap v23: 97 pgs: 97 active+clean; 316 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 424 op/s 2024-09-13T21:43:09.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 67971 10.63 MB/sec execute 22 sec latency 58.829 ms 2024-09-13T21:43:10.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 68286 10.40 MB/sec execute 23 sec latency 74.383 ms 2024-09-13T21:43:11.286 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:11 smithi154 ceph-mon[23700]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 625 op/s 2024-09-13T21:43:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:11 smithi181 ceph-mon[25811]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 625 op/s 2024-09-13T21:43:11.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 68541 10.11 MB/sec execute 24 sec latency 135.662 ms 2024-09-13T21:43:12.296 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:12 smithi154 ceph-mon[23700]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-13T21:43:12.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:12 smithi181 ceph-mon[25811]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-13T21:43:12.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 69011 10.18 MB/sec execute 25 sec latency 75.097 ms 2024-09-13T21:43:13.323 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:13 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:43:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:13 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:43:13.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 69549 10.24 MB/sec execute 26 sec latency 48.127 ms 2024-09-13T21:43:14.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:14 smithi154 ceph-mon[23700]: pgmap v26: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-13T21:43:14.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:14 smithi181 ceph-mon[25811]: pgmap v26: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-13T21:43:14.854 INFO:teuthology.orchestra.run.smithi154.stdout: 5 70013 10.31 MB/sec execute 27 sec latency 53.607 ms 2024-09-13T21:43:15.609 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:15 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:15.609 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:15 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:15.609 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:15 smithi154 ceph-mon[23700]: Reconfiguring prometheus.smithi154 (dependencies changed)... 2024-09-13T21:43:15.609 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:15 smithi154 ceph-mon[23700]: Deploying daemon prometheus.smithi154 on smithi154 2024-09-13T21:43:15.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 70453 10.57 MB/sec execute 28 sec latency 75.652 ms 2024-09-13T21:43:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:15 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:15 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:15 smithi181 ceph-mon[25811]: Reconfiguring prometheus.smithi154 (dependencies changed)... 2024-09-13T21:43:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:15 smithi181 ceph-mon[25811]: Deploying daemon prometheus.smithi154 on smithi154 2024-09-13T21:43:16.638 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:16 smithi154 ceph-mon[23700]: pgmap v27: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 640 op/s 2024-09-13T21:43:16.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 70911 10.76 MB/sec execute 29 sec latency 130.048 ms 2024-09-13T21:43:16.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:16 smithi181 ceph-mon[25811]: pgmap v27: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 640 op/s 2024-09-13T21:43:17.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 71369 10.86 MB/sec execute 30 sec latency 75.746 ms 2024-09-13T21:43:18.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 71855 10.81 MB/sec execute 31 sec latency 36.641 ms 2024-09-13T21:43:19.153 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:18 smithi154 ceph-mon[23700]: pgmap v28: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 423 op/s 2024-09-13T21:43:19.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:18 smithi181 ceph-mon[25811]: pgmap v28: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 423 op/s 2024-09-13T21:43:19.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 72305 10.75 MB/sec execute 32 sec latency 30.434 ms 2024-09-13T21:43:20.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 72877 10.89 MB/sec execute 33 sec latency 27.498 ms 2024-09-13T21:43:21.426 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:21 smithi154 ceph-mon[23700]: pgmap v29: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.4 MiB/s wr, 690 op/s 2024-09-13T21:43:21.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:21 smithi181 ceph-mon[25811]: pgmap v29: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.4 MiB/s wr, 690 op/s 2024-09-13T21:43:21.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 73432 10.94 MB/sec execute 34 sec latency 33.076 ms 2024-09-13T21:43:22.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 73984 11.11 MB/sec execute 35 sec latency 42.230 ms 2024-09-13T21:43:23.445 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:23 smithi154 ceph-mon[23700]: pgmap v30: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 489 op/s 2024-09-13T21:43:23.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:23 smithi181 ceph-mon[25811]: pgmap v30: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 489 op/s 2024-09-13T21:43:23.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 74618 11.42 MB/sec execute 36 sec latency 28.589 ms 2024-09-13T21:43:24.593 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:43:24.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 75287 11.62 MB/sec execute 37 sec latency 23.062 ms 2024-09-13T21:43:25.205 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:25 smithi154 ceph-mon[23700]: pgmap v31: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 489 op/s 2024-09-13T21:43:25.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:25 smithi181 ceph-mon[25811]: pgmap v31: 97 pgs: 97 active+clean; 328 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 489 op/s 2024-09-13T21:43:25.733 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:43:25.733 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 starting - - - - 2024-09-13T21:43:25.733 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (9m) 46s ago 9m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:43:25.733 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (8m) 51s ago 8m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:43:25.733 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 starting - - - - 2024-09-13T21:43:25.733 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (3m) 46s ago 3m 34.0M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:43:25.733 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (3m) 51s ago 3m 15.4M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:9283 running (10m) 46s ago 10m 168M - 17.2.0 e1d6a67b021e c729166e328e 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283 running (68s) 51s ago 8m 478M - 19.1.1-324-gecab7be5 0652ff2b5d78 01d71e511aaf 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (10m) 46s ago 10m 52.7M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (8m) 51s ago 8m 40.0M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (2m) 46s ago 2m 253M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (2m) 51s ago 2m 71.4M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 starting - - - - 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (8m) 51s ago 8m 17.0M - 1.3.1 1dbe0e931976 31f26a6812e0 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (7m) 46s ago 7m 250M 1936M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (7m) 46s ago 7m 183M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (6m) 46s ago 6m 157M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (6m) 46s ago 6m 195M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:43:25.734 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (5m) 51s ago 5m 218M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:43:25.735 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (5m) 51s ago 5m 212M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:43:25.735 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (5m) 51s ago 5m 200M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:43:25.735 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (4m) 51s ago 4m 297M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:43:25.735 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (8m) 46s ago 8m 65.4M - 2.33.4 514e6a882f6e ca18cf24d925 2024-09-13T21:43:25.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 75874 11.57 MB/sec execute 38 sec latency 35.803 ms 2024-09-13T21:43:26.217 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:26 smithi154 ceph-mon[23700]: from='client.24477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:26.218 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:26 smithi154 ceph-mon[23700]: from='client.24481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:43:26.448 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:43:26.449 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:43:26.449 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:43:26.449 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:43:26.449 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:43:26.449 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T21:43:26.449 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T21:43:26.449 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:43:26.449 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:43:26.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:26 smithi181 ceph-mon[25811]: from='client.24477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:26.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:26 smithi181 ceph-mon[25811]: from='client.24481 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:26.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 76445 11.67 MB/sec execute 39 sec latency 28.542 ms 2024-09-13T21:43:27.055 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:43:27.055 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:43:27.056 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:43:27.056 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:43:27.056 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [], 2024-09-13T21:43:27.056 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T21:43:27.056 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "", 2024-09-13T21:43:27.056 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:43:27.056 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:43:27.225 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:27 smithi154 ceph-mon[23700]: from='client.24485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:27.225 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:27 smithi154 ceph-mon[23700]: pgmap v32: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 814 op/s 2024-09-13T21:43:27.225 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:27 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/375503666' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:43:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:27 smithi181 ceph-mon[25811]: from='client.24485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:27 smithi181 ceph-mon[25811]: pgmap v32: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 814 op/s 2024-09-13T21:43:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:27 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/375503666' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:43:27.718 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:43:27.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 77049 11.71 MB/sec execute 40 sec latency 31.691 ms 2024-09-13T21:43:28.247 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:28 smithi154 ceph-mon[23700]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:28.247 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:28 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1333757931' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:43:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:28 smithi181 ceph-mon[25811]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:28 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1333757931' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:43:28.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 77555 11.82 MB/sec execute 41 sec latency 29.496 ms 2024-09-13T21:43:29.466 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:29 smithi154 ceph-mon[23700]: pgmap v33: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 592 op/s 2024-09-13T21:43:29.466 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:29 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:43:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:29 smithi181 ceph-mon[25811]: pgmap v33: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 592 op/s 2024-09-13T21:43:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:29 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:43:29.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 78011 11.93 MB/sec execute 42 sec latency 34.998 ms 2024-09-13T21:43:30.856 INFO:teuthology.orchestra.run.smithi154.stdout: 5 78384 11.96 MB/sec execute 43 sec latency 65.289 ms 2024-09-13T21:43:31.495 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:31 smithi154 ceph-mon[23700]: pgmap v34: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 938 op/s 2024-09-13T21:43:31.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:31 smithi181 ceph-mon[25811]: pgmap v34: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.9 MiB/s wr, 938 op/s 2024-09-13T21:43:31.855 INFO:teuthology.orchestra.run.smithi154.stdout: 5 78695 11.91 MB/sec execute 44 sec latency 80.972 ms 2024-09-13T21:43:32.861 INFO:teuthology.orchestra.run.smithi154.stdout: 5 79006 11.71 MB/sec execute 45 sec latency 119.005 ms 2024-09-13T21:43:33.291 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:33 smithi154 ceph-mon[23700]: pgmap v35: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 670 op/s 2024-09-13T21:43:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:33 smithi181 ceph-mon[25811]: pgmap v35: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 670 op/s 2024-09-13T21:43:33.860 INFO:teuthology.orchestra.run.smithi154.stdout: 5 79278 11.56 MB/sec execute 46 sec latency 176.643 ms 2024-09-13T21:43:34.860 INFO:teuthology.orchestra.run.smithi154.stdout: 5 79683 11.52 MB/sec execute 47 sec latency 48.082 ms 2024-09-13T21:43:35.316 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:35 smithi154 ceph-mon[23700]: pgmap v36: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 670 op/s 2024-09-13T21:43:35.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:35 smithi181 ceph-mon[25811]: pgmap v36: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 670 op/s 2024-09-13T21:43:35.860 INFO:teuthology.orchestra.run.smithi154.stdout: 5 80187 11.53 MB/sec execute 48 sec latency 55.630 ms 2024-09-13T21:43:36.326 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:36 smithi154 ceph-mon[23700]: pgmap v37: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 887 op/s 2024-09-13T21:43:36.550 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:36 smithi181 ceph-mon[25811]: pgmap v37: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 887 op/s 2024-09-13T21:43:36.860 INFO:teuthology.orchestra.run.smithi154.stdout: 5 80884 11.74 MB/sec execute 49 sec latency 32.893 ms 2024-09-13T21:43:37.861 INFO:teuthology.orchestra.run.smithi154.stdout: 5 81536 11.94 MB/sec execute 50 sec latency 28.845 ms 2024-09-13T21:43:38.587 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:38 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:38.587 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:38 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:38.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:38 smithi154 ceph-mon[23700]: Reconfiguring node-exporter.smithi181 (dependencies changed)... 2024-09-13T21:43:38.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:38 smithi154 ceph-mon[23700]: Deploying daemon node-exporter.smithi181 on smithi181 2024-09-13T21:43:38.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:38 smithi154 ceph-mon[23700]: pgmap v38: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 562 op/s 2024-09-13T21:43:38.615 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:38 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:38.615 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:38 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:38.615 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:38 smithi181 ceph-mon[25811]: Reconfiguring node-exporter.smithi181 (dependencies changed)... 2024-09-13T21:43:38.615 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:38 smithi181 ceph-mon[25811]: Deploying daemon node-exporter.smithi181 on smithi181 2024-09-13T21:43:38.615 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:38 smithi181 ceph-mon[25811]: pgmap v38: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 562 op/s 2024-09-13T21:43:38.861 INFO:teuthology.orchestra.run.smithi154.stdout: 5 82116 12.02 MB/sec execute 51 sec latency 33.034 ms 2024-09-13T21:43:39.861 INFO:teuthology.orchestra.run.smithi154.stdout: 5 82596 11.99 MB/sec execute 52 sec latency 40.697 ms 2024-09-13T21:43:40.861 INFO:teuthology.orchestra.run.smithi154.stdout: 5 83405 12.15 MB/sec execute 53 sec latency 35.329 ms 2024-09-13T21:43:41.367 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:41 smithi154 ceph-mon[23700]: pgmap v39: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 889 op/s 2024-09-13T21:43:41.485 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:41 smithi181 ceph-mon[25811]: pgmap v39: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 889 op/s 2024-09-13T21:43:41.869 INFO:teuthology.orchestra.run.smithi154.stdout: 5 84249 12.33 MB/sec execute 54 sec latency 35.151 ms 2024-09-13T21:43:42.869 INFO:teuthology.orchestra.run.smithi154.stdout: 5 84652 12.29 MB/sec execute 55 sec latency 40.694 ms 2024-09-13T21:43:43.382 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: pgmap v40: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 543 op/s 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi154.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi154.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi154.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.383 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:43 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:43:43.472 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: pgmap v40: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 543 op/s 2024-09-13T21:43:43.472 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi154.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi154.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi154.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:43.473 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:43 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:43:43.869 INFO:teuthology.orchestra.run.smithi154.stdout: 5 85145 12.38 MB/sec execute 56 sec latency 43.181 ms 2024-09-13T21:43:44.406 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:43:44.406 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi154.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T21:43:44.406 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:43:44.407 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi154.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T21:43:44.407 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:43:44.407 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi154.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T21:43:44.407 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: Upgrade: Need to upgrade myself (mgr.smithi181.byvkib) 2024-09-13T21:43:44.407 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: Upgrade: Need to upgrade myself (mgr.smithi181.byvkib) 2024-09-13T21:43:44.407 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:44 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi154.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi154.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi154.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: Upgrade: Need to upgrade myself (mgr.smithi181.byvkib) 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: Upgrade: Need to upgrade myself (mgr.smithi181.byvkib) 2024-09-13T21:43:44.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:44 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:43:44.869 INFO:teuthology.orchestra.run.smithi154.stdout: 5 85636 12.43 MB/sec execute 57 sec latency 27.494 ms 2024-09-13T21:43:45.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:45 smithi181 ceph-mon[25811]: pgmap v41: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 543 op/s 2024-09-13T21:43:45.416 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:45 smithi154 ceph-mon[23700]: pgmap v41: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 543 op/s 2024-09-13T21:43:45.869 INFO:teuthology.orchestra.run.smithi154.stdout: 5 86050 12.38 MB/sec execute 58 sec latency 38.345 ms 2024-09-13T21:43:46.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:46 smithi154 ceph-mon[23700]: Upgrade: Updating mgr.smithi154.ayrwdi 2024-09-13T21:43:46.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:46 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:46.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:46 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.ayrwdi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:43:46.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:46 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.ayrwdi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:43:46.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:46 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:43:46.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:46 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:43:46.427 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:46 smithi154 ceph-mon[23700]: Deploying daemon mgr.smithi154.ayrwdi on smithi154 2024-09-13T21:43:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:46 smithi181 ceph-mon[25811]: Upgrade: Updating mgr.smithi154.ayrwdi 2024-09-13T21:43:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:46 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:46 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.ayrwdi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:43:46.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:46 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.ayrwdi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:43:46.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:46 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:43:46.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:46 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:43:46.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:46 smithi181 ceph-mon[25811]: Deploying daemon mgr.smithi154.ayrwdi on smithi154 2024-09-13T21:43:46.869 INFO:teuthology.orchestra.run.smithi154.stdout: 5 86452 12.31 MB/sec execute 59 sec latency 43.858 ms 2024-09-13T21:43:47.442 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:47 smithi154 ceph-mon[23700]: pgmap v42: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 890 op/s 2024-09-13T21:43:47.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:47 smithi181 ceph-mon[25811]: pgmap v42: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 890 op/s 2024-09-13T21:43:47.869 INFO:teuthology.orchestra.run.smithi154.stdout: 5 86885 12.27 MB/sec execute 60 sec latency 43.883 ms 2024-09-13T21:43:48.870 INFO:teuthology.orchestra.run.smithi154.stdout: 5 87339 12.23 MB/sec execute 61 sec latency 47.599 ms 2024-09-13T21:43:49.295 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:49 smithi154 ceph-mon[23700]: pgmap v43: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.1 MiB/s wr, 674 op/s 2024-09-13T21:43:49.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:49 smithi181 ceph-mon[25811]: pgmap v43: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.1 MiB/s wr, 674 op/s 2024-09-13T21:43:49.869 INFO:teuthology.orchestra.run.smithi154.stdout: 5 87819 12.30 MB/sec execute 62 sec latency 51.205 ms 2024-09-13T21:43:50.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 88403 12.37 MB/sec execute 63 sec latency 38.066 ms 2024-09-13T21:43:51.475 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:51 smithi154 ceph-mon[23700]: pgmap v44: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.7 MiB/s wr, 948 op/s 2024-09-13T21:43:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:51 smithi181 ceph-mon[25811]: pgmap v44: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.7 MiB/s wr, 948 op/s 2024-09-13T21:43:51.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 88868 12.41 MB/sec execute 64 sec latency 31.635 ms 2024-09-13T21:43:52.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:52 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:52.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:52 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:52.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:52 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:43:52.562 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:52 smithi154 ceph-mon[23700]: pgmap v45: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-13T21:43:52.562 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:52 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:52.562 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:52 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:52.846 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:52 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:52.846 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:52 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:52.847 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:52 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:43:52.847 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:52 smithi181 ceph-mon[25811]: pgmap v45: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-13T21:43:52.847 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:52 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:52.847 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:52 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:52.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 89264 12.42 MB/sec execute 65 sec latency 31.671 ms 2024-09-13T21:43:53.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 89700 12.34 MB/sec execute 66 sec latency 40.243 ms 2024-09-13T21:43:54.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 90324 12.41 MB/sec execute 67 sec latency 21.639 ms 2024-09-13T21:43:55.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:54 smithi181 ceph-mon[25811]: pgmap v46: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-13T21:43:55.085 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:54 smithi154 ceph-mon[23700]: pgmap v46: 97 pgs: 97 active+clean; 319 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-13T21:43:55.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 90801 12.37 MB/sec execute 68 sec latency 31.487 ms 2024-09-13T21:43:56.372 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:56 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:56.372 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:56 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:56 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:56 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:43:56.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 91197 12.33 MB/sec execute 69 sec latency 42.405 ms 2024-09-13T21:43:57.396 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:57 smithi154 ceph-mon[23700]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 897 op/s 2024-09-13T21:43:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:57 smithi181 ceph-mon[25811]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.1 MiB/s wr, 897 op/s 2024-09-13T21:43:57.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 91618 12.33 MB/sec execute 70 sec latency 60.019 ms 2024-09-13T21:43:58.258 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:43:58.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:58 smithi154 ceph-mon[23700]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-13T21:43:58.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:58 smithi154 ceph-mon[23700]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:58.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:58 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:43:58.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:58 smithi181 ceph-mon[25811]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-13T21:43:58.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:58 smithi181 ceph-mon[25811]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:58.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:58 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:43:58.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 92004 12.35 MB/sec execute 71 sec latency 45.382 ms 2024-09-13T21:43:59.280 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:43:59.280 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 starting - - - - 2024-09-13T21:43:59.280 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (9m) 79s ago 9m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:43:59.280 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (9m) 4s ago 9m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 starting - - - - 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (3m) 79s ago 3m 34.0M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (3m) 4s ago 3m 15.7M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 starting - - - - 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283 running (102s) 4s ago 8m 502M - 19.1.1-324-gecab7be5 0652ff2b5d78 01d71e511aaf 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (10m) 79s ago 10m 52.7M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (8m) 4s ago 8m 42.3M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (3m) 79s ago 3m 253M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (3m) 4s ago 3m 78.0M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 starting - - - - 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (17s) 4s ago 8m 7172k - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (8m) 79s ago 8m 250M 1936M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:43:59.281 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (7m) 79s ago 7m 183M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:43:59.282 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (7m) 79s ago 7m 157M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:43:59.282 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (6m) 79s ago 6m 195M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:43:59.282 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (6m) 4s ago 6m 338M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:43:59.282 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (6m) 4s ago 6m 344M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:43:59.282 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (5m) 4s ago 5m 292M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:43:59.282 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (5m) 4s ago 5m 432M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:43:59.282 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 starting - - - - 2024-09-13T21:43:59.429 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:59 smithi154 ceph-mon[23700]: Standby manager daemon smithi154.ayrwdi restarted 2024-09-13T21:43:59.429 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:59 smithi154 ceph-mon[23700]: Standby manager daemon smithi154.ayrwdi started 2024-09-13T21:43:59.429 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:59 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.ayrwdi/crt"}]: dispatch 2024-09-13T21:43:59.430 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:59 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:43:59.430 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:59 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.ayrwdi/key"}]: dispatch 2024-09-13T21:43:59.430 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:59 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:43:59.430 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:43:59 smithi154 ceph-mon[23700]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:59 smithi181 ceph-mon[25811]: Standby manager daemon smithi154.ayrwdi restarted 2024-09-13T21:43:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:59 smithi181 ceph-mon[25811]: Standby manager daemon smithi154.ayrwdi started 2024-09-13T21:43:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:59 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.ayrwdi/crt"}]: dispatch 2024-09-13T21:43:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:59 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:43:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:59 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.ayrwdi/key"}]: dispatch 2024-09-13T21:43:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:59 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:43:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:43:59 smithi181 ceph-mon[25811]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:43:59.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 92324 12.30 MB/sec execute 72 sec latency 64.149 ms 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:43:59.911 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:43:59.912 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:43:59.912 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:43:59.912 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:43:59.912 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T21:43:59.912 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:43:59.912 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:43:59.912 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:44:00.433 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:44:00.433 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:44:00.433 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:44:00.433 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:44:00.433 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [], 2024-09-13T21:44:00.433 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "", 2024-09-13T21:44:00.434 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T21:44:00.434 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:44:00.434 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:44:00.443 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:00 smithi154 ceph-mon[23700]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:00.443 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:00 smithi154 ceph-mon[23700]: mgrmap e28: smithi181.byvkib(active, since 92s), standbys: smithi154.ayrwdi 2024-09-13T21:44:00.443 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:00 smithi154 ceph-mon[23700]: pgmap v49: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 817 op/s 2024-09-13T21:44:00.443 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:00 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/3228376376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:44:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:00 smithi181 ceph-mon[25811]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:00 smithi181 ceph-mon[25811]: mgrmap e28: smithi181.byvkib(active, since 92s), standbys: smithi154.ayrwdi 2024-09-13T21:44:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:00 smithi181 ceph-mon[25811]: pgmap v49: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 817 op/s 2024-09-13T21:44:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:00 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/3228376376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:44:00.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 92746 12.35 MB/sec execute 73 sec latency 54.581 ms 2024-09-13T21:44:01.005 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:44:01.878 INFO:teuthology.orchestra.run.smithi154.stdout: 5 93045 12.26 MB/sec execute 74 sec latency 69.680 ms 2024-09-13T21:44:02.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:01 smithi154 ceph-mon[23700]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:02.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:01 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:02.223 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:01 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/970072851' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:44:02.224 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:01 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:02.224 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:01 smithi154 ceph-mon[23700]: mgrmap e29: smithi181.byvkib(active, since 94s), standbys: smithi154.ayrwdi 2024-09-13T21:44:02.224 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:01 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:02.224 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:01 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:01 smithi181 ceph-mon[25811]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:01 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:01 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/970072851' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:44:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:01 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:01 smithi181 ceph-mon[25811]: mgrmap e29: smithi181.byvkib(active, since 94s), standbys: smithi154.ayrwdi 2024-09-13T21:44:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:01 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:01 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:02.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 93441 12.17 MB/sec execute 75 sec latency 108.488 ms 2024-09-13T21:44:03.240 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:02 smithi154 ceph-mon[23700]: pgmap v50: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 543 op/s 2024-09-13T21:44:03.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:02 smithi181 ceph-mon[25811]: pgmap v50: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 543 op/s 2024-09-13T21:44:03.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 94098 12.23 MB/sec execute 76 sec latency 24.567 ms 2024-09-13T21:44:04.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 94604 12.25 MB/sec execute 77 sec latency 40.347 ms 2024-09-13T21:44:05.016 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:05 smithi154 ceph-mon[23700]: pgmap v51: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-13T21:44:05.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:05 smithi181 ceph-mon[25811]: pgmap v51: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-13T21:44:05.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 95052 12.28 MB/sec execute 78 sec latency 29.783 ms 2024-09-13T21:44:06.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 95498 12.28 MB/sec execute 79 sec latency 44.785 ms 2024-09-13T21:44:07.298 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:07 smithi154 ceph-mon[23700]: pgmap v52: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 804 op/s 2024-09-13T21:44:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:07 smithi181 ceph-mon[25811]: pgmap v52: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 804 op/s 2024-09-13T21:44:07.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 95783 12.25 MB/sec execute 80 sec latency 85.739 ms 2024-09-13T21:44:08.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 96127 12.28 MB/sec execute 81 sec latency 35.199 ms 2024-09-13T21:44:09.345 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: pgmap v53: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 528 op/s 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr fail", "who": "smithi181.byvkib"}]: dispatch 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr fail", "who": "smithi181.byvkib"}]: dispatch 2024-09-13T21:44:09.346 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:09 smithi154 ceph-mon[23700]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T21:44:09.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: pgmap v53: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 528 op/s 2024-09-13T21:44:09.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:09.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:09.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:09.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:44:09.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:09.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' 2024-09-13T21:44:09.510 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:09.510 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 172.21.15.181:0/1276381477' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr fail", "who": "smithi181.byvkib"}]: dispatch 2024-09-13T21:44:09.510 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "mgr fail", "who": "smithi181.byvkib"}]: dispatch 2024-09-13T21:44:09.510 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:09 smithi181 ceph-mon[25811]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T21:44:09.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 96491 12.18 MB/sec execute 82 sec latency 43.334 ms 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: Upgrade: Need to upgrade myself (mgr.smithi181.byvkib) 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: Upgrade: Need to upgrade myself (mgr.smithi181.byvkib) 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: Failing over to other MGR 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd='[{"prefix": "mgr fail", "who": "smithi181.byvkib"}]': finished 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: mgrmap e30: smithi154.ayrwdi(active, starting, since 0.863807s) 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:44:10.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:44:10.360 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:10.361 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:44:10.361 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:44:10.361 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:10 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:44:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: Upgrade: Need to upgrade myself (mgr.smithi181.byvkib) 2024-09-13T21:44:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: Upgrade: Need to upgrade myself (mgr.smithi181.byvkib) 2024-09-13T21:44:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: Failing over to other MGR 2024-09-13T21:44:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.24411 ' entity='mgr.smithi181.byvkib' cmd='[{"prefix": "mgr fail", "who": "smithi181.byvkib"}]': finished 2024-09-13T21:44:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: mgrmap e30: smithi154.ayrwdi(active, starting, since 0.863807s) 2024-09-13T21:44:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:44:10.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:44:10.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:44:10.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:10.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:44:10.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:44:10.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:10 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:44:10.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 96804 12.13 MB/sec execute 83 sec latency 50.857 ms 2024-09-13T21:44:11.665 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:11 smithi154 ceph-mon[23700]: mgrmap e31: smithi154.ayrwdi(active, since 1.86929s) 2024-09-13T21:44:11.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:11 smithi154 ceph-mon[23700]: pgmap v3: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T21:44:11.879 INFO:teuthology.orchestra.run.smithi154.stdout: 5 97036 12.05 MB/sec execute 84 sec latency 79.753 ms 2024-09-13T21:44:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:11 smithi181 ceph-mon[25811]: mgrmap e31: smithi154.ayrwdi(active, since 1.86929s) 2024-09-13T21:44:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:11 smithi181 ceph-mon[25811]: pgmap v3: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T21:44:12.690 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:12 smithi154 ceph-mon[23700]: mgrmap e32: smithi154.ayrwdi(active, since 2s) 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:12 smithi154 ceph-mon[23700]: [13/Sep/2024:21:44:11] ENGINE Bus STARTING 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:12 smithi154 ceph-mon[23700]: [13/Sep/2024:21:44:11] ENGINE Serving on https://172.21.15.154:7150 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:12 smithi154 ceph-mon[23700]: [13/Sep/2024:21:44:11] ENGINE Error in HTTPServer.serve 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Traceback (most recent call last): 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._connections.run(self.expiration_interval) 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._run(expiration_interval) 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T21:44:12.691 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: s = self.context.wrap_socket( 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: return self.sslsocket_class._create( 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self.do_handshake() 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._sslobj.do_handshake() 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:12 smithi154 ceph-mon[23700]: [13/Sep/2024:21:44:11] ENGINE Serving on http://172.21.15.154:8765 2024-09-13T21:44:12.692 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:12 smithi154 ceph-mon[23700]: [13/Sep/2024:21:44:11] ENGINE Bus STARTED 2024-09-13T21:44:12.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 97386 12.03 MB/sec execute 85 sec latency 65.906 ms 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:12 smithi181 ceph-mon[25811]: mgrmap e32: smithi154.ayrwdi(active, since 2s) 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:12 smithi181 ceph-mon[25811]: [13/Sep/2024:21:44:11] ENGINE Bus STARTING 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:12 smithi181 ceph-mon[25811]: [13/Sep/2024:21:44:11] ENGINE Serving on https://172.21.15.154:7150 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:12 smithi181 ceph-mon[25811]: [13/Sep/2024:21:44:11] ENGINE Error in HTTPServer.serve 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Traceback (most recent call last): 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._connections.run(self.expiration_interval) 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._run(expiration_interval) 2024-09-13T21:44:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: s = self.context.wrap_socket( 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: return self.sslsocket_class._create( 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self.do_handshake() 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._sslobj.do_handshake() 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:12 smithi181 ceph-mon[25811]: [13/Sep/2024:21:44:11] ENGINE Serving on http://172.21.15.154:8765 2024-09-13T21:44:12.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:12 smithi181 ceph-mon[25811]: [13/Sep/2024:21:44:11] ENGINE Bus STARTED 2024-09-13T21:44:13.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 97728 11.95 MB/sec execute 86 sec latency 44.553 ms 2024-09-13T21:44:14.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:14 smithi154 ceph-mon[23700]: pgmap v5: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T21:44:14.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:14 smithi154 ceph-mon[23700]: mgrmap e33: smithi154.ayrwdi(active, since 4s) 2024-09-13T21:44:14.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 98084 11.92 MB/sec execute 87 sec latency 46.954 ms 2024-09-13T21:44:14.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:14 smithi181 ceph-mon[25811]: pgmap v5: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T21:44:14.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:14 smithi181 ceph-mon[25811]: mgrmap e33: smithi154.ayrwdi(active, since 4s) 2024-09-13T21:44:15.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:15.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:15.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:44:15.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/3605904253' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:44:15.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/3605904253' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:44:15.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/3605904253' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:44:15.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/3605904253' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:44:15.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:15.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:15 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:15.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 98415 11.91 MB/sec execute 88 sec latency 53.420 ms 2024-09-13T21:44:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:44:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/3605904253' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:44:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/3605904253' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:44:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/3605904253' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:44:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/3605904253' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:44:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:16.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:15 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:16.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 98799 11.87 MB/sec execute 89 sec latency 51.491 ms 2024-09-13T21:44:17.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:17 smithi154 ceph-mon[23700]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s wr, 11 op/s 2024-09-13T21:44:17.300 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:17 smithi154 ceph-mon[23700]: mgrmap e34: smithi154.ayrwdi(active, since 7s), standbys: smithi181.byvkib 2024-09-13T21:44:17.300 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:17 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi181.byvkib", "id": "smithi181.byvkib"}]: dispatch 2024-09-13T21:44:17.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:17 smithi181 ceph-mon[25811]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s wr, 11 op/s 2024-09-13T21:44:17.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:17 smithi181 ceph-mon[25811]: mgrmap e34: smithi154.ayrwdi(active, since 7s), standbys: smithi181.byvkib 2024-09-13T21:44:17.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:17 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi181.byvkib", "id": "smithi181.byvkib"}]: dispatch 2024-09-13T21:44:17.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 99173 11.89 MB/sec execute 90 sec latency 56.428 ms 2024-09-13T21:44:18.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 99435 11.85 MB/sec execute 91 sec latency 66.612 ms 2024-09-13T21:44:19.338 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:19 smithi154 ceph-mon[23700]: pgmap v7: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 3.1 MiB/s wr, 363 op/s 2024-09-13T21:44:19.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:19 smithi181 ceph-mon[25811]: pgmap v7: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 3.1 MiB/s wr, 363 op/s 2024-09-13T21:44:19.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 99703 11.85 MB/sec execute 92 sec latency 71.407 ms 2024-09-13T21:44:20.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 100018 11.77 MB/sec execute 93 sec latency 49.358 ms 2024-09-13T21:44:21.133 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:21 smithi154 ceph-mon[23700]: pgmap v8: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.4 MiB/s wr, 282 op/s 2024-09-13T21:44:21.134 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:21 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:21.134 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:21 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:21.134 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:21 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:21.134 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:21 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:21 smithi181 ceph-mon[25811]: pgmap v8: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.4 MiB/s wr, 282 op/s 2024-09-13T21:44:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:21 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:21 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:21 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:21 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:21.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 100402 11.73 MB/sec execute 94 sec latency 79.842 ms 2024-09-13T21:44:22.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 100745 11.71 MB/sec execute 95 sec latency 63.500 ms 2024-09-13T21:44:23.228 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:23 smithi154 ceph-mon[23700]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-13T21:44:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:23 smithi181 ceph-mon[25811]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-13T21:44:23.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 101201 11.68 MB/sec execute 96 sec latency 38.580 ms 2024-09-13T21:44:24.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 101579 11.67 MB/sec execute 97 sec latency 41.226 ms 2024-09-13T21:44:25.268 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:25 smithi154 ceph-mon[23700]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-13T21:44:25.268 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:25 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:44:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:25 smithi181 ceph-mon[25811]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-13T21:44:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:25 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:44:25.880 INFO:teuthology.orchestra.run.smithi154.stdout: 5 101979 11.68 MB/sec execute 98 sec latency 80.462 ms 2024-09-13T21:44:26.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 102315 11.64 MB/sec execute 99 sec latency 62.017 ms 2024-09-13T21:44:27.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:27 smithi154 ceph-mon[23700]: pgmap v11: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.6 MiB/s wr, 597 op/s 2024-09-13T21:44:27.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:27 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:27.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:27 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:27.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:27 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:27.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:27 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:44:27.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:27 smithi181 ceph-mon[25811]: pgmap v11: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.6 MiB/s wr, 597 op/s 2024-09-13T21:44:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:27 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:27 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:27 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:27 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:44:27.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 102674 11.64 MB/sec execute 100 sec latency 44.404 ms 2024-09-13T21:44:28.329 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: Updating smithi181:/etc/ceph/ceph.conf 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.330 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:28 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: Updating smithi181:/etc/ceph/ceph.conf 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:28 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:28.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 102966 11.62 MB/sec execute 101 sec latency 43.540 ms 2024-09-13T21:44:29.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:29 smithi154 ceph-mon[23700]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:44:29.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:29 smithi154 ceph-mon[23700]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:44:29.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:29 smithi154 ceph-mon[23700]: pgmap v12: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.2 MiB/s wr, 633 op/s 2024-09-13T21:44:29.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:29 smithi154 ceph-mon[23700]: Reconfiguring prometheus.smithi154 (dependencies changed)... 2024-09-13T21:44:29.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:29 smithi154 ceph-mon[23700]: Reconfiguring daemon prometheus.smithi154 on smithi154 2024-09-13T21:44:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:29 smithi181 ceph-mon[25811]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:44:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:29 smithi181 ceph-mon[25811]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:44:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:29 smithi181 ceph-mon[25811]: pgmap v12: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.2 MiB/s wr, 633 op/s 2024-09-13T21:44:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:29 smithi181 ceph-mon[25811]: Reconfiguring prometheus.smithi154 (dependencies changed)... 2024-09-13T21:44:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:29 smithi181 ceph-mon[25811]: Reconfiguring daemon prometheus.smithi154 on smithi154 2024-09-13T21:44:29.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 103274 11.63 MB/sec execute 102 sec latency 50.942 ms 2024-09-13T21:44:30.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 103545 11.55 MB/sec execute 103 sec latency 87.628 ms 2024-09-13T21:44:31.401 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:31 smithi154 ceph-mon[23700]: pgmap v13: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 428 op/s 2024-09-13T21:44:31.523 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:44:31.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:31 smithi181 ceph-mon[25811]: pgmap v13: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 428 op/s 2024-09-13T21:44:31.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 103853 11.51 MB/sec execute 104 sec latency 59.381 ms 2024-09-13T21:44:32.422 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:44:32.422 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (94s) 12s ago 10m 28.8M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:44:32.422 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (10m) 12s ago 10m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:44:32.422 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (9m) 17s ago 9m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (86s) 12s ago 9m 43.9M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (4m) 12s ago 4m 36.1M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (4m) 17s ago 4m 16.0M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (41s) 12s ago 11m 474M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283 running (2m) 17s ago 9m 113M - 19.1.1-324-gecab7be5 0652ff2b5d78 01d71e511aaf 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (11m) 12s ago 11m 57.2M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (9m) 17s ago 9m 44.6M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (3m) 12s ago 3m 262M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (3m) 17s ago 3m 79.9M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (78s) 12s ago 9m 7520k - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (50s) 17s ago 9m 7629k - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (8m) 12s ago 8m 387M 1936M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (8m) 12s ago 8m 287M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:44:32.423 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (7m) 12s ago 7m 262M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:44:32.424 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (7m) 12s ago 7m 359M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:44:32.424 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (6m) 17s ago 6m 352M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:44:32.424 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (6m) 17s ago 6m 385M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:44:32.424 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (6m) 17s ago 6m 322M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:44:32.424 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (5m) 17s ago 5m 453M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:44:32.424 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (55s) 12s ago 9m 46.8M - 2.43.0 a07b618ecd1d b0338745397c 2024-09-13T21:44:32.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 104034 11.42 MB/sec execute 105 sec latency 132.017 ms 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:44:32.932 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:44:32.933 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:44:33.249 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:33 smithi154 ceph-mon[23700]: from='client.24549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:33.249 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:33 smithi154 ceph-mon[23700]: pgmap v14: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 621 op/s 2024-09-13T21:44:33.249 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:33 smithi154 ceph-mon[23700]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:33.250 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:33 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/757458352' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:44:33.439 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr" 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "", 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:44:33.440 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:44:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:33 smithi181 ceph-mon[25811]: from='client.24549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:33 smithi181 ceph-mon[25811]: pgmap v14: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 621 op/s 2024-09-13T21:44:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:33 smithi181 ceph-mon[25811]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:33 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/757458352' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:44:33.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 104423 11.42 MB/sec execute 106 sec latency 48.670 ms 2024-09-13T21:44:34.018 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:44:34.269 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:34 smithi154 ceph-mon[23700]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:34.269 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:34 smithi154 ceph-mon[23700]: from='client.? 172.21.15.154:0/1724469346' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:44:34.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:34 smithi181 ceph-mon[25811]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:34.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:34 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/1724469346' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:44:34.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 104839 11.41 MB/sec execute 107 sec latency 103.069 ms 2024-09-13T21:44:35.306 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:35 smithi154 ceph-mon[23700]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:35.306 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:35 smithi154 ceph-mon[23700]: pgmap v15: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-09-13T21:44:35.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:35 smithi181 ceph-mon[25811]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:44:35.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:35 smithi181 ceph-mon[25811]: pgmap v15: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-09-13T21:44:35.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 105238 11.38 MB/sec execute 108 sec latency 65.686 ms 2024-09-13T21:44:36.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 105525 11.38 MB/sec execute 109 sec latency 71.865 ms 2024-09-13T21:44:37.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:37 smithi154 ceph-mon[23700]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-09-13T21:44:37.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:37 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:37.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:37 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:37.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:37 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:44:37.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:37 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:37.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:37 smithi181 ceph-mon[25811]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-09-13T21:44:37.452 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:37 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:37.452 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:37 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:37.452 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:37 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:44:37.452 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:37 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:37.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 105996 11.38 MB/sec execute 110 sec latency 42.902 ms 2024-09-13T21:44:38.347 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:38 smithi154 ceph-mon[23700]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:44:38.544 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:38 smithi181 ceph-mon[25811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T21:44:38.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 106381 11.39 MB/sec execute 111 sec latency 44.741 ms 2024-09-13T21:44:39.355 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:39 smithi154 ceph-mon[23700]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.6 MiB/s wr, 435 op/s 2024-09-13T21:44:39.355 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:39 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:44:39.355 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:39 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:44:39.355 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:39 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:39 smithi181 ceph-mon[25811]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.6 MiB/s wr, 435 op/s 2024-09-13T21:44:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:39 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:44:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:39 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:44:39.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:39 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:39.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 106759 11.43 MB/sec execute 112 sec latency 43.711 ms 2024-09-13T21:44:40.381 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:40 smithi154 ceph-mon[23700]: Upgrade: Updating mgr.smithi181.byvkib 2024-09-13T21:44:40.381 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:40 smithi154 ceph-mon[23700]: Deploying daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:44:40.381 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:40 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:44:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:40 smithi181 ceph-mon[25811]: Upgrade: Updating mgr.smithi181.byvkib 2024-09-13T21:44:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:40 smithi181 ceph-mon[25811]: Deploying daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:44:40.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:40 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:44:40.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 107082 11.38 MB/sec execute 113 sec latency 52.132 ms 2024-09-13T21:44:41.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:41 smithi181 ceph-mon[25811]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-13T21:44:41.403 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:41 smithi154 ceph-mon[23700]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-13T21:44:41.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 107458 11.34 MB/sec execute 114 sec latency 47.917 ms 2024-09-13T21:44:42.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 107712 11.30 MB/sec execute 115 sec latency 56.231 ms 2024-09-13T21:44:43.373 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:43 smithi181 ceph-mon[25811]: pgmap v19: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.5 MiB/s wr, 613 op/s 2024-09-13T21:44:43.373 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:43 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:43.373 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:43 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:43.373 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:43 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:43.452 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:43 smithi154 ceph-mon[23700]: pgmap v19: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.5 MiB/s wr, 613 op/s 2024-09-13T21:44:43.452 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:43 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:43.452 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:43 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:43.452 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:43 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:43.881 INFO:teuthology.orchestra.run.smithi154.stdout: 5 107988 11.26 MB/sec execute 116 sec latency 87.470 ms 2024-09-13T21:44:44.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 108339 11.25 MB/sec execute 117 sec latency 70.364 ms 2024-09-13T21:44:45.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:45 smithi154 ceph-mon[23700]: pgmap v20: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-13T21:44:45.600 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:45 smithi181 ceph-mon[25811]: pgmap v20: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-13T21:44:45.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 108667 11.21 MB/sec execute 118 sec latency 65.322 ms 2024-09-13T21:44:46.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 109021 11.20 MB/sec execute 119 sec latency 30.870 ms 2024-09-13T21:44:47.503 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:47 smithi154 ceph-mon[23700]: pgmap v21: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 433 op/s 2024-09-13T21:44:47.503 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:47 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:47.503 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:47 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:47.505 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:47 smithi181 ceph-mon[25811]: pgmap v21: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 433 op/s 2024-09-13T21:44:47.505 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:47 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:47.505 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:47 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:47.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 109274 11.16 MB/sec execute 120 sec latency 70.080 ms 2024-09-13T21:44:48.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 109647 11.18 MB/sec execute 121 sec latency 56.023 ms 2024-09-13T21:44:49.046 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:48 smithi154 ceph-mon[23700]: pgmap v22: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.3 MiB/s wr, 547 op/s 2024-09-13T21:44:49.046 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:48 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:49.046 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:48 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:49.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:48 smithi181 ceph-mon[25811]: pgmap v22: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.3 MiB/s wr, 547 op/s 2024-09-13T21:44:49.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:48 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:49.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:48 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:49.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 109966 11.15 MB/sec execute 122 sec latency 67.274 ms 2024-09-13T21:44:50.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:49 smithi154 ceph-mon[23700]: Standby manager daemon smithi181.byvkib restarted 2024-09-13T21:44:50.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:49 smithi154 ceph-mon[23700]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:44:50.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:49 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/2006397592' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:44:50.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:49 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/2006397592' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:44:50.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:49 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/2006397592' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:44:50.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:49 smithi154 ceph-mon[23700]: from='mgr.? 172.21.15.181:0/2006397592' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:44:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:49 smithi181 ceph-mon[25811]: Standby manager daemon smithi181.byvkib restarted 2024-09-13T21:44:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:49 smithi181 ceph-mon[25811]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:44:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:49 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/2006397592' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:44:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:49 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/2006397592' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:44:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:49 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/2006397592' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:44:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:49 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/2006397592' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:44:50.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 110330 11.18 MB/sec execute 123 sec latency 62.069 ms 2024-09-13T21:44:51.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:50 smithi154 ceph-mon[23700]: pgmap v23: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-13T21:44:51.108 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:50 smithi154 ceph-mon[23700]: mgrmap e35: smithi154.ayrwdi(active, since 41s), standbys: smithi181.byvkib 2024-09-13T21:44:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:50 smithi181 ceph-mon[25811]: pgmap v23: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-13T21:44:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:50 smithi181 ceph-mon[25811]: mgrmap e35: smithi154.ayrwdi(active, since 41s), standbys: smithi181.byvkib 2024-09-13T21:44:51.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 110644 11.14 MB/sec execute 124 sec latency 73.720 ms 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:44:52.149 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.150 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi154.ayrwdi"}]: dispatch 2024-09-13T21:44:52.150 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi154.ayrwdi"}]': finished 2024-09-13T21:44:52.150 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.byvkib"}]: dispatch 2024-09-13T21:44:52.150 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.byvkib"}]': finished 2024-09-13T21:44:52.150 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:52 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T21:44:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:44:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi154.ayrwdi"}]: dispatch 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi154.ayrwdi"}]': finished 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.byvkib"}]: dispatch 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.byvkib"}]': finished 2024-09-13T21:44:52.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:52 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T21:44:52.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 111012 11.11 MB/sec execute 125 sec latency 41.149 ms 2024-09-13T21:44:53.157 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:53 smithi154 ceph-mon[23700]: Upgrade: Setting container_image for all mgr 2024-09-13T21:44:53.157 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:53 smithi154 ceph-mon[23700]: pgmap v24: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.5 MiB/s wr, 610 op/s 2024-09-13T21:44:53.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:53 smithi181 ceph-mon[25811]: Upgrade: Setting container_image for all mgr 2024-09-13T21:44:53.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:53 smithi181 ceph-mon[25811]: pgmap v24: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.5 MiB/s wr, 610 op/s 2024-09-13T21:44:53.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 111307 11.07 MB/sec execute 126 sec latency 38.848 ms 2024-09-13T21:44:54.885 INFO:teuthology.orchestra.run.smithi154.stdout: 5 111790 11.11 MB/sec execute 127 sec latency 45.017 ms 2024-09-13T21:44:54.932 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:54 smithi154 ceph-mon[23700]: pgmap v25: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 391 op/s 2024-09-13T21:44:54.933 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:54 smithi154 ceph-mon[23700]: Upgrade: Updating mon.smithi154 2024-09-13T21:44:54.933 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:54 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:54.933 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:54 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:44:54.933 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:54 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:44:54.933 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:54 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:54.933 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:54 smithi154 ceph-mon[23700]: Deploying daemon mon.smithi154 on smithi154 2024-09-13T21:44:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:54 smithi181 ceph-mon[25811]: pgmap v25: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 391 op/s 2024-09-13T21:44:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:54 smithi181 ceph-mon[25811]: Upgrade: Updating mon.smithi154 2024-09-13T21:44:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:54 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:44:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:54 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:44:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:54 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:44:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:54 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:44:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:54 smithi181 ceph-mon[25811]: Deploying daemon mon.smithi154 on smithi154 2024-09-13T21:44:55.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 112153 11.07 MB/sec execute 128 sec latency 55.462 ms 2024-09-13T21:44:55.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:55 smithi154 ceph-mon[23700]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:44:56.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:55 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:44:56.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 112729 11.12 MB/sec execute 129 sec latency 65.182 ms 2024-09-13T21:44:57.215 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:56 smithi154 systemd[1]: Stopping Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:44:57.472 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:57 smithi154 ceph-mon[23700]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.3 MiB/s wr, 574 op/s 2024-09-13T21:44:57.472 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:57 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[23696]: 2024-09-13T21:44:57.382+0000 7f37f7cd4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi154 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:44:57.472 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:57 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[23696]: 2024-09-13T21:44:57.382+0000 7f37f7cd4700 -1 mon.smithi154@0(leader) e2 *** Got Signal Terminated *** 2024-09-13T21:44:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:44:57 smithi181 ceph-mon[25811]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.3 MiB/s wr, 574 op/s 2024-09-13T21:44:57.722 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:57 smithi154 podman[65270]: 2024-09-13 21:44:57.47485422 +0000 UTC m=+0.236886082 container died ca544568ebc042e7dcd70d02923dfc3b9d56f779e7a4cef0931bce03429266ad (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, version=8, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.buildah.version=1.19.8, architecture=x86_64, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-13T21:44:57.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 113381 11.18 MB/sec execute 130 sec latency 49.914 ms 2024-09-13T21:44:58.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:58 smithi154 podman[65270]: 2024-09-13 21:44:58.354672606 +0000 UTC m=+1.116704463 container cleanup ca544568ebc042e7dcd70d02923dfc3b9d56f779e7a4cef0931bce03429266ad (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, distribution-scope=public, release=754, io.buildah.version=1.19.8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, version=8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True) 2024-09-13T21:44:58.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:58 smithi154 bash[65270]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154 2024-09-13T21:44:58.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 113703 11.17 MB/sec execute 131 sec latency 53.741 ms 2024-09-13T21:44:59.240 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:59 smithi154 podman[65283]: 2024-09-13 21:44:59.136514092 +0000 UTC m=+1.660841880 container remove ca544568ebc042e7dcd70d02923dfc3b9d56f779e7a4cef0931bce03429266ad (image=quay.io/ceph/ceph:v17.2.0, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, name=centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD) 2024-09-13T21:44:59.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:59 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154.service: Deactivated successfully. 2024-09-13T21:44:59.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:59 smithi154 systemd[1]: Stopped Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:44:59.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:59 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154.service: Consumed 15.021s CPU time. 2024-09-13T21:44:59.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 114033 11.16 MB/sec execute 132 sec latency 43.380 ms 2024-09-13T21:45:00.000 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:44:59 smithi154 systemd[1]: Starting Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:45:00.255 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 podman[65417]: 2024-09-13 21:44:59.95623389 +0000 UTC m=+0.020896989 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:45:00.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 podman[65417]: 2024-09-13 21:45:00.116561408 +0000 UTC m=+0.181224509 container create 44ff9ab2ade69caf91969ce179e435d14bc4979a1c852441dd4c25690c9dee5a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T21:45:00.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 podman[65417]: 2024-09-13 21:45:00.509728663 +0000 UTC m=+0.574391784 container init 44ff9ab2ade69caf91969ce179e435d14bc4979a1c852441dd4c25690c9dee5a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T21:45:00.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 podman[65417]: 2024-09-13 21:45:00.521109695 +0000 UTC m=+0.585772804 container start 44ff9ab2ade69caf91969ce179e435d14bc4979a1c852441dd4c25690c9dee5a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T21:45:00.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T21:45:00.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T21:45:00.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: pidfile_write: ignore empty --pid-file 2024-09-13T21:45:00.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: load: jerasure load: lrc 2024-09-13T21:45:00.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: RocksDB version: 7.9.2 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Git sha 0 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: DB SUMMARY 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: DB Session ID: BE1SPYRFP62AWNL0BZGA 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: CURRENT file: CURRENT 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: MANIFEST file: MANIFEST-000015 size: 980 Bytes 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi154/store.db dir, Total Num: 1, files: 000030.sst 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi154/store.db: 000028.log size: 842012 ; 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.error_if_exists: 0 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.create_if_missing: 0 2024-09-13T21:45:00.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.paranoid_checks: 1 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.env: 0x556395f25940 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.info_log: 0x5563985218c0 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.statistics: (nil) 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.use_fsync: 0 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_log_file_size: 0 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T21:45:00.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.allow_fallocate: 1 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.use_direct_reads: 0 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.db_log_dir: 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.wal_dir: 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T21:45:00.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.write_buffer_manager: 0x556398530dc0 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.unordered_write: 0 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T21:45:00.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.row_cache: None 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.wal_filter: None 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.two_write_queues: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.wal_compression: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.atomic_flush: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T21:45:00.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.log_readahead_size: 0 2024-09-13T21:45:00.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T21:45:00.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T21:45:00.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T21:45:00.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_background_jobs: 2 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_background_compactions: -1 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_subcompactions: 1 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T21:45:00.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_open_files: -1 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_background_flushes: -1 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Compression algorithms supported: 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: kZSTD supported: 0 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: kXpressCompression supported: 0 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: kBZip2Compression supported: 0 2024-09-13T21:45:00.771 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: kLZ4Compression supported: 1 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: kZlibCompression supported: 1 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: kSnappyCompression supported: 1 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi154/store.db/MANIFEST-000015 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.merge_operator: 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_filter: None 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T21:45:00.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556398521520) 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: cache_index_and_filter_blocks: 1 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: pin_top_level_index_and_filter: 1 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_type: 0 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: data_block_index_type: 0 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_shortening: 1 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: checksum: 4 2024-09-13T21:45:00.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: no_block_cache: 0 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache: 0x5563985151f0 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_name: BinnedLRUCache 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_options: 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: capacity : 536870912 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: num_shard_bits : 4 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: strict_capacity_limit : 0 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: high_pri_pool_ratio: 0.000 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_compressed: (nil) 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: persistent_cache: (nil) 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_size: 4096 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_size_deviation: 10 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_restart_interval: 16 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_block_restart_interval: 1 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: metadata_block_size: 4096 2024-09-13T21:45:00.774 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: partition_filters: 0 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: use_delta_encoding: 1 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: filter_policy: bloomfilter 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: whole_key_filtering: 1 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: verify_compression: 0 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: read_amp_bytes_per_bit: 0 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: format_version: 5 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: enable_index_compression: 1 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_align: 0 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: max_auto_readahead_size: 262144 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: prepopulate_block_cache: 0 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: initial_auto_readahead_size: 8192 2024-09-13T21:45:00.775 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T21:45:00.776 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T21:45:00.776 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression: NoCompression 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.num_levels: 7 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:45:00.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T21:45:00.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T21:45:00.779 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T21:45:00.780 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.inplace_update_support: 0 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.bloom_locality: 0 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.max_successive_merges: 0 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T21:45:00.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.ttl: 2592000 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.enable_blob_files: false 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.min_blob_size: 0 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T21:45:00.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi154/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11900, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3e873461-9702-4c19-a304-7156db38235b 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263900557849, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263900562992, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 770341, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11884, "largest_seqno": 12427, "table_properties": {"data_size": 767148, "index_size": 1513, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 645, "raw_key_size": 5958, "raw_average_key_size": 24, "raw_value_size": 761776, "raw_average_value_size": 3109, "num_data_blocks": 68, "num_entries": 245, "num_filter_entries": 245, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726263900, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3e873461-9702-4c19-a304-7156db38235b", "db_session_id": "BE1SPYRFP62AWNL0BZGA", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263900563196, "job": 1, "event": "recovery_finished"} 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T21:45:00.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi154/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556398542e00 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: DB pointer 0x55639863e000 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** DB Stats ** 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: L0 1/0 752.29 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 247.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: L6 1/0 7.60 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Sum 2/0 8.33 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 247.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 247.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 247.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T21:45:00.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative compaction: 0.00 GB write, 82.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval compaction: 0.00 GB write, 82.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Block cache BinnedLRUCache@0x5563985151f0#2 capacity: 512.00 MB usage: 2.30 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.72 KB,0.000137091%) IndexBlock(1,1.58 KB,0.000301003%) Misc(1,0.00 KB,0%) 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:45:00.786 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: starting mon.smithi154 rank 0 at public addrs [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] at bind addrs [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi154 fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: mon.smithi154@-1(???) e2 preinit fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: mon.smithi154@-1(???).mds e7 new map 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: mon.smithi154@-1(???).mds e7 print_map 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: e7 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: legacy client fscid: 1 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Filesystem 'foofs' (1) 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: fs_name foofs 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: epoch 5 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T21:45:00.787 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: created 2024-09-13T21:40:09.703677+0000 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: modified 2024-09-13T21:40:15.728417+0000 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: tableserver 0 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: root 0 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: session_timeout 60 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: session_autoclose 300 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: max_file_size 1099511627776 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: max_xattr_size 65536 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: required_client_features {} 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: last_failure 0 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: last_failure_osd_epoch 0 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: max_mds 1 2024-09-13T21:45:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: in 0 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: up {0=14498} 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: failed 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: damaged 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: stopped 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: data_pools [3] 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: metadata_pool 2 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: inline_data disabled 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: balancer 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: bal_rank_mask -1 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: standby_count_wanted 1 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: qdb_cluster leader: 0 members: 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: [mds.foofs.smithi154.uynwpr{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.154:6834/3259978861,v1:172.21.15.154:6835/3259978861] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Standby daemons: 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: [mds.foofs.smithi181.mgvpzf{-1:24269} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: mon.smithi154@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: mon.smithi154@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: mon.smithi154@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: mon.smithi154@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 ceph-mon[65435]: mon.smithi154@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 bash[65417]: 44ff9ab2ade69caf91969ce179e435d14bc4979a1c852441dd4c25690c9dee5a 2024-09-13T21:45:00.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:00 smithi154 systemd[1]: Started Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:45:00.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 114410 11.12 MB/sec execute 133 sec latency 84.450 ms 2024-09-13T21:45:01.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 114957 11.16 MB/sec execute 134 sec latency 84.512 ms 2024-09-13T21:45:02.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 115154 11.11 MB/sec execute 135 sec latency 660.972 ms 2024-09-13T21:45:03.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 115207 11.03 MB/sec execute 136 sec latency 1497.289 ms 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 606 op/s 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: pgmap v29: 97 pgs: 97 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.2 MiB/s wr, 692 op/s 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: mon.smithi154 calling monitor election 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: mon.smithi154 is new leader, mons smithi154,smithi181 in quorum (ranks 0,1) 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: monmap epoch 2 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: last_changed 2024-09-13T21:35:04.934281+0000 2024-09-13T21:45:04.029 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: created 2024-09-13T21:32:51.022089+0000 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: min_mon_release 17 (quincy) 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: election_strategy: 1 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: 0: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.smithi154 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: 1: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.smithi181 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: mgrmap e35: smithi154.ayrwdi(active, since 55s), standbys: smithi181.byvkib 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: overall HEALTH_OK 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: from='mgr.14752 ' entity='' 2024-09-13T21:45:04.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:03 smithi154 ceph-mon[65435]: mgrmap e36: smithi154.ayrwdi(active, since 55s), standbys: smithi181.byvkib 2024-09-13T21:45:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:45:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: from='mgr.14752 172.21.15.154:0/4205760020' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:45:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 606 op/s 2024-09-13T21:45:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-09-13T21:45:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: pgmap v29: 97 pgs: 97 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.2 MiB/s wr, 692 op/s 2024-09-13T21:45:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: mon.smithi154 calling monitor election 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: mon.smithi154 is new leader, mons smithi154,smithi181 in quorum (ranks 0,1) 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: monmap epoch 2 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: last_changed 2024-09-13T21:35:04.934281+0000 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: created 2024-09-13T21:32:51.022089+0000 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: min_mon_release 17 (quincy) 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: election_strategy: 1 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: 0: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.smithi154 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: 1: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.smithi181 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: mgrmap e35: smithi154.ayrwdi(active, since 55s), standbys: smithi181.byvkib 2024-09-13T21:45:04.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: overall HEALTH_OK 2024-09-13T21:45:04.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: from='mgr.14752 ' entity='' 2024-09-13T21:45:04.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:03 smithi181 ceph-mon[25811]: mgrmap e36: smithi154.ayrwdi(active, since 55s), standbys: smithi181.byvkib 2024-09-13T21:45:04.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 115446 11.01 MB/sec execute 137 sec latency 69.499 ms 2024-09-13T21:45:05.824 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:05 smithi154 ceph-mon[65435]: mgrmap e37: smithi154.ayrwdi(active, since 56s), standbys: smithi181.byvkib 2024-09-13T21:45:05.882 INFO:teuthology.orchestra.run.smithi154.stdout: 5 115855 10.99 MB/sec execute 138 sec latency 51.848 ms 2024-09-13T21:45:06.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:05 smithi181 ceph-mon[25811]: mgrmap e37: smithi154.ayrwdi(active, since 56s), standbys: smithi181.byvkib 2024-09-13T21:45:06.883 INFO:teuthology.orchestra.run.smithi154.stdout: 5 116126 10.96 MB/sec execute 139 sec latency 69.379 ms 2024-09-13T21:45:07.883 INFO:teuthology.orchestra.run.smithi154.stdout: 5 116580 11.01 MB/sec execute 140 sec latency 78.112 ms 2024-09-13T21:45:08.883 INFO:teuthology.orchestra.run.smithi154.stdout: 5 117191 11.05 MB/sec execute 141 sec latency 58.448 ms 2024-09-13T21:45:09.883 INFO:teuthology.orchestra.run.smithi154.stdout: 5 117492 11.02 MB/sec execute 142 sec latency 60.209 ms 2024-09-13T21:45:09.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:09 smithi154 ceph-mon[65435]: Standby manager daemon smithi181.byvkib restarted 2024-09-13T21:45:09.899 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:09 smithi154 ceph-mon[65435]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:45:09.899 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:09 smithi154 ceph-mon[65435]: from='mgr.? 172.21.15.181:0/238676273' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:45:09.899 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:09 smithi154 ceph-mon[65435]: from='mgr.? 172.21.15.181:0/238676273' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:45:09.899 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:09 smithi154 ceph-mon[65435]: from='mgr.? 172.21.15.181:0/238676273' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:45:09.899 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:09 smithi154 ceph-mon[65435]: from='mgr.? 172.21.15.181:0/238676273' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:45:09.961 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:09 smithi181 ceph-mon[25811]: Standby manager daemon smithi181.byvkib restarted 2024-09-13T21:45:09.962 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:09 smithi181 ceph-mon[25811]: Standby manager daemon smithi181.byvkib started 2024-09-13T21:45:09.962 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:09 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/238676273' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/crt"}]: dispatch 2024-09-13T21:45:09.962 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:09 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/238676273' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T21:45:09.962 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:09 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/238676273' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.byvkib/key"}]: dispatch 2024-09-13T21:45:09.962 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:09 smithi181 ceph-mon[25811]: from='mgr.? 172.21.15.181:0/238676273' entity='mgr.smithi181.byvkib' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T21:45:10.883 INFO:teuthology.orchestra.run.smithi154.stdout: 5 117829 11.00 MB/sec execute 143 sec latency 68.175 ms 2024-09-13T21:45:10.922 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: mgrmap e38: smithi154.ayrwdi(active, since 61s), standbys: smithi181.byvkib 2024-09-13T21:45:10.922 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: Active manager daemon smithi154.ayrwdi restarted 2024-09-13T21:45:10.922 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: Activating manager daemon smithi154.ayrwdi 2024-09-13T21:45:10.922 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T21:45:10.922 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: mgrmap e39: smithi154.ayrwdi(active, starting, since 0.00742952s), standbys: smithi181.byvkib 2024-09-13T21:45:10.922 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:45:10.922 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi181.byvkib", "id": "smithi181.byvkib"}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:45:10.923 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:45:10.924 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:45:10.924 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:45:10.924 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:45:10.924 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:45:10.924 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:45:10.924 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:45:10.924 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:45:11.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: mgrmap e38: smithi154.ayrwdi(active, since 61s), standbys: smithi181.byvkib 2024-09-13T21:45:11.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: Active manager daemon smithi154.ayrwdi restarted 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: Activating manager daemon smithi154.ayrwdi 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: mgrmap e39: smithi154.ayrwdi(active, starting, since 0.00742952s), standbys: smithi181.byvkib 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi154.ayrwdi", "id": "smithi154.ayrwdi"}]: dispatch 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr metadata", "who": "smithi181.byvkib", "id": "smithi181.byvkib"}]: dispatch 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:45:11.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: Manager daemon smithi154.ayrwdi is now available 2024-09-13T21:45:11.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:45:11.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:45:11.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/mirror_snapshot_schedule"}]: dispatch 2024-09-13T21:45:11.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:10 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi154.ayrwdi/trash_purge_schedule"}]: dispatch 2024-09-13T21:45:11.433 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:45:11.883 INFO:teuthology.orchestra.run.smithi154.stdout: 5 118206 10.98 MB/sec execute 144 sec latency 42.740 ms 2024-09-13T21:45:12.444 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:12 smithi154 ceph-mon[65435]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:12.444 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:12 smithi154 ceph-mon[65435]: mgrmap e40: smithi154.ayrwdi(active, since 1.01569s), standbys: smithi181.byvkib 2024-09-13T21:45:12.444 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:12 smithi154 ceph-mon[65435]: pgmap v3: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T21:45:12.444 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:12 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:12.444 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:12 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:12.497 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:45:12.500 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (2m) 21s ago 10m 28.8M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:45:12.500 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (10m) 21s ago 10m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:45:12.500 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (10m) 25s ago 10m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:45:12.500 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (2m) 21s ago 10m 48.1M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (4m) 21s ago 4m 35.8M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (4m) 25s ago 4m 16.1M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (81s) 21s ago 12m 492M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (30s) 25s ago 10m 68.3M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (12m) 21s ago 12m 52.9M 2048M 17.2.0 e1d6a67b021e ca544568ebc0 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (10m) 25s ago 10m 40.0M 2048M 17.2.0 e1d6a67b021e 690acc2b7d67 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (4m) 21s ago 4m 261M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (4m) 25s ago 4m 80.0M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (118s) 21s ago 10m 7700k - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:45:12.501 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (90s) 25s ago 9m 8094k - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (9m) 21s ago 9m 430M 1936M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (8m) 21s ago 8m 320M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (8m) 21s ago 8m 290M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (8m) 21s ago 8m 396M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (7m) 25s ago 7m 376M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (7m) 25s ago 7m 420M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (6m) 25s ago 6m 348M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (6m) 25s ago 6m 488M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:45:12.502 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (36s) 21s ago 9m 43.0M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:45:12.829 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:12 smithi181 ceph-mon[25811]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:12.829 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:12 smithi181 ceph-mon[25811]: mgrmap e40: smithi154.ayrwdi(active, since 1.01569s), standbys: smithi181.byvkib 2024-09-13T21:45:12.829 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:12 smithi181 ceph-mon[25811]: pgmap v3: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T21:45:12.829 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:12 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:12.829 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:12 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:12.883 INFO:teuthology.orchestra.run.smithi154.stdout: 5 118619 10.99 MB/sec execute 145 sec latency 39.048 ms 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:45:13.084 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:45:13.085 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:45:13.519 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr" 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "", 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:45:13.520 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:45:13.703 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: [13/Sep/2024:21:45:12] ENGINE Bus STARTING 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: [13/Sep/2024:21:45:12] ENGINE Serving on https://172.21.15.154:7150 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: [13/Sep/2024:21:45:12] ENGINE Error in HTTPServer.serve 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Traceback (most recent call last): 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._connections.run(self.expiration_interval) 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._run(expiration_interval) 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T21:45:13.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: s = self.context.wrap_socket( 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: return self.sslsocket_class._create( 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self.do_handshake() 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: self._sslobj.do_handshake() 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: pgmap v4: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: mgrmap e41: smithi154.ayrwdi(active, since 2s), standbys: smithi181.byvkib 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: from='client.24591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: [13/Sep/2024:21:45:12] ENGINE Serving on http://172.21.15.154:8765 2024-09-13T21:45:13.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: [13/Sep/2024:21:45:12] ENGINE Bus STARTED 2024-09-13T21:45:13.706 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:13 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2232241688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: [13/Sep/2024:21:45:12] ENGINE Bus STARTING 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: [13/Sep/2024:21:45:12] ENGINE Serving on https://172.21.15.154:7150 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: [13/Sep/2024:21:45:12] ENGINE Error in HTTPServer.serve 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Traceback (most recent call last): 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._connections.run(self.expiration_interval) 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._run(expiration_interval) 2024-09-13T21:45:13.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: s = self.context.wrap_socket( 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: return self.sslsocket_class._create( 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self.do_handshake() 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: self._sslobj.do_handshake() 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T21:45:13.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: pgmap v4: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T21:45:13.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: mgrmap e41: smithi154.ayrwdi(active, since 2s), standbys: smithi181.byvkib 2024-09-13T21:45:13.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: from='client.24591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:13.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: [13/Sep/2024:21:45:12] ENGINE Serving on http://172.21.15.154:8765 2024-09-13T21:45:13.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: [13/Sep/2024:21:45:12] ENGINE Bus STARTED 2024-09-13T21:45:13.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:13 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2232241688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:45:13.883 INFO:teuthology.orchestra.run.smithi154.stdout: 5 119040 11.01 MB/sec execute 146 sec latency 72.571 ms 2024-09-13T21:45:14.061 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:45:14.726 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:14 smithi154 ceph-mon[65435]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:14.726 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:14 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2143249988' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:45:14.816 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:14 smithi181 ceph-mon[25811]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:14.816 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:14 smithi181 ceph-mon[25811]: from='client.? 172.21.15.154:0/2143249988' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:45:14.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 119421 10.98 MB/sec execute 147 sec latency 154.537 ms 2024-09-13T21:45:15.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:15 smithi154 ceph-mon[65435]: pgmap v5: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T21:45:15.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:15 smithi154 ceph-mon[65435]: mgrmap e42: smithi154.ayrwdi(active, since 4s), standbys: smithi181.byvkib 2024-09-13T21:45:15.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:15.737 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:15.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 119771 10.96 MB/sec execute 148 sec latency 29.254 ms 2024-09-13T21:45:15.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:15 smithi181 ceph-mon[25811]: pgmap v5: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T21:45:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:15 smithi181 ceph-mon[25811]: mgrmap e42: smithi154.ayrwdi(active, since 4s), standbys: smithi181.byvkib 2024-09-13T21:45:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:15 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:15 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:16.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 120596 11.09 MB/sec execute 149 sec latency 26.353 ms 2024-09-13T21:45:17.244 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:17 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:17.244 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:17 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:17.244 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:17 smithi154 ceph-mon[65435]: pgmap v6: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T21:45:17.314 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:17 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:17.315 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:17 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:17.315 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:17 smithi181 ceph-mon[25811]: pgmap v6: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T21:45:17.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 121040 11.07 MB/sec execute 150 sec latency 46.001 ms 2024-09-13T21:45:18.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 121444 11.05 MB/sec execute 151 sec latency 36.505 ms 2024-09-13T21:45:19.514 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:19 smithi154 ceph-mon[65435]: pgmap v7: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-09-13T21:45:19.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 121729 11.03 MB/sec execute 152 sec latency 48.483 ms 2024-09-13T21:45:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:19 smithi181 ceph-mon[25811]: pgmap v7: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-09-13T21:45:20.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 122361 11.10 MB/sec execute 153 sec latency 66.340 ms 2024-09-13T21:45:21.042 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.043 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.043 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.043 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.043 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: pgmap v8: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.6 MiB/s wr, 361 op/s 2024-09-13T21:45:21.043 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.043 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.043 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:21.043 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:45:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: pgmap v8: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.6 MiB/s wr, 361 op/s 2024-09-13T21:45:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:21.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:21.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:20 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:45:21.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 123025 11.11 MB/sec execute 154 sec latency 34.811 ms 2024-09-13T21:45:22.461 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:45:22.461 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: Updating smithi181:/etc/ceph/ceph.conf 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.462 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:22 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.conf 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: Updating smithi181:/etc/ceph/ceph.conf 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: Updating smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:22 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:22.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 123353 11.09 MB/sec execute 155 sec latency 40.540 ms 2024-09-13T21:45:23.494 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:23 smithi154 ceph-mon[65435]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:45:23.495 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:45:23.495 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:45:23.495 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T21:45:23.495 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:23 smithi154 ceph-mon[65435]: pgmap v9: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-13T21:45:23.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:23 smithi181 ceph-mon[25811]: Updating smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T21:45:23.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:23 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:45:23.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:23 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:45:23.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:23 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T21:45:23.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:23 smithi181 ceph-mon[25811]: pgmap v9: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-13T21:45:23.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 123723 11.12 MB/sec execute 156 sec latency 41.521 ms 2024-09-13T21:45:24.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 124056 11.09 MB/sec execute 157 sec latency 65.639 ms 2024-09-13T21:45:25.048 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:25 smithi154 ceph-mon[65435]: Upgrade: Updating mon.smithi181 2024-09-13T21:45:25.049 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:25.049 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:45:25.049 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:45:25.049 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:25.049 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:25 smithi154 ceph-mon[65435]: Deploying daemon mon.smithi181 on smithi181 2024-09-13T21:45:25.049 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:25 smithi154 ceph-mon[65435]: pgmap v10: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 5.6 MiB/s wr, 568 op/s 2024-09-13T21:45:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:25 smithi181 ceph-mon[25811]: Upgrade: Updating mon.smithi181 2024-09-13T21:45:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:25 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:25 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:45:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:25 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:45:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:25 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:25 smithi181 ceph-mon[25811]: Deploying daemon mon.smithi181 on smithi181 2024-09-13T21:45:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:25 smithi181 ceph-mon[25811]: pgmap v10: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 5.6 MiB/s wr, 568 op/s 2024-09-13T21:45:25.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 124448 11.10 MB/sec execute 158 sec latency 49.094 ms 2024-09-13T21:45:26.056 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:26 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:45:26.142 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:25 smithi181 systemd[1]: Stopping Ceph mon.smithi181 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:45:26.142 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:26 smithi181 ceph-mon[25811]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:45:26.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:26 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181[25793]: 2024-09-13T21:45:26.138+0000 7fc3ec337700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi181 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:45:26.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:26 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181[25793]: 2024-09-13T21:45:26.138+0000 7fc3ec337700 -1 mon.smithi181@1(peon) e2 *** Got Signal Terminated *** 2024-09-13T21:45:26.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:26 smithi181 podman[51517]: 2024-09-13 21:45:26.211222594 +0000 UTC m=+0.201031345 container died 690acc2b7d671bde24b1f4dcf40fc9f9d1c9f49bde35dbe3bf86dc9ad57583f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, architecture=x86_64, ceph=True, name=centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, release=754, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-type=git, build-date=2022-05-03T08:36:31.336870, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8) 2024-09-13T21:45:26.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 124721 11.05 MB/sec execute 159 sec latency 46.172 ms 2024-09-13T21:45:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:26 smithi181 podman[51517]: 2024-09-13 21:45:26.856869458 +0000 UTC m=+0.846678210 container cleanup 690acc2b7d671bde24b1f4dcf40fc9f9d1c9f49bde35dbe3bf86dc9ad57583f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, RELEASE=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T21:45:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:26 smithi181 bash[51517]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181 2024-09-13T21:45:27.537 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:27 smithi181 podman[51528]: 2024-09-13 21:45:27.150369918 +0000 UTC m=+0.937639202 container remove 690acc2b7d671bde24b1f4dcf40fc9f9d1c9f49bde35dbe3bf86dc9ad57583f9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, GIT_CLEAN=True, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, release=754, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, name=centos-stream, RELEASE=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-13T21:45:27.538 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:27 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi181.service: Deactivated successfully. 2024-09-13T21:45:27.538 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:27 smithi181 systemd[1]: Stopped Ceph mon.smithi181 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:45:27.538 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:27 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi181.service: Consumed 6.783s CPU time. 2024-09-13T21:45:27.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 125026 11.03 MB/sec execute 160 sec latency 48.988 ms 2024-09-13T21:45:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:27 smithi181 systemd[1]: Starting Ceph mon.smithi181 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:45:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:27 smithi181 podman[51657]: 2024-09-13 21:45:27.658733799 +0000 UTC m=+0.017837311 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:45:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:27 smithi181 podman[51657]: 2024-09-13 21:45:27.785363319 +0000 UTC m=+0.144466883 container create cf973304eaffa760c4aee5b19c947336ddf76ef6bdd7c42f0b9c21775adcd9f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T21:45:28.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 125362 11.03 MB/sec execute 161 sec latency 55.975 ms 2024-09-13T21:45:28.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 podman[51657]: 2024-09-13 21:45:28.538922096 +0000 UTC m=+0.898025607 container init cf973304eaffa760c4aee5b19c947336ddf76ef6bdd7c42f0b9c21775adcd9f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T21:45:28.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 podman[51657]: 2024-09-13 21:45:28.543717382 +0000 UTC m=+0.902820883 container start cf973304eaffa760c4aee5b19c947336ddf76ef6bdd7c42f0b9c21775adcd9f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T21:45:28.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T21:45:28.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T21:45:28.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: pidfile_write: ignore empty --pid-file 2024-09-13T21:45:28.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: load: jerasure load: lrc 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: RocksDB version: 7.9.2 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Git sha 0 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: DB SUMMARY 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: DB Session ID: EOJOTHJB86G8KAWDH7O9 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: CURRENT file: CURRENT 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: MANIFEST file: MANIFEST-000009 size: 905 Bytes 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi181/store.db dir, Total Num: 1, files: 000024.sst 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi181/store.db: 000022.log size: 4578299 ; 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.error_if_exists: 0 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.create_if_missing: 0 2024-09-13T21:45:28.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.paranoid_checks: 1 2024-09-13T21:45:28.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T21:45:28.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T21:45:28.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T21:45:28.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.env: 0x556fc322a940 2024-09-13T21:45:28.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T21:45:28.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.info_log: 0x556fc531b8c0 2024-09-13T21:45:28.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T21:45:28.903 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.statistics: (nil) 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.use_fsync: 0 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_log_file_size: 0 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.allow_fallocate: 1 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.use_direct_reads: 0 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T21:45:28.904 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.db_log_dir: 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.wal_dir: 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.write_buffer_manager: 0x556fc532adc0 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T21:45:28.905 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.unordered_write: 0 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.row_cache: None 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.wal_filter: None 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T21:45:28.906 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.two_write_queues: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.wal_compression: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.atomic_flush: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.log_readahead_size: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T21:45:28.907 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_background_jobs: 2 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_background_compactions: -1 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_subcompactions: 1 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_open_files: -1 2024-09-13T21:45:28.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_background_flushes: -1 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Compression algorithms supported: 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: kZSTD supported: 0 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: kXpressCompression supported: 0 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: kBZip2Compression supported: 0 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: kLZ4Compression supported: 1 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: kZlibCompression supported: 1 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: kSnappyCompression supported: 1 2024-09-13T21:45:28.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi181/store.db/MANIFEST-000009 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.merge_operator: 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_filter: None 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556fc531b520) 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: cache_index_and_filter_blocks: 1 2024-09-13T21:45:28.910 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: pin_top_level_index_and_filter: 1 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_type: 0 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_block_index_type: 0 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_shortening: 1 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: checksum: 4 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: no_block_cache: 0 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache: 0x556fc530f1f0 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_name: BinnedLRUCache 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_options: 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: capacity : 536870912 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: num_shard_bits : 4 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: strict_capacity_limit : 0 2024-09-13T21:45:28.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: high_pri_pool_ratio: 0.000 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_compressed: (nil) 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: persistent_cache: (nil) 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_size: 4096 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_size_deviation: 10 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_restart_interval: 16 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_block_restart_interval: 1 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: metadata_block_size: 4096 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: partition_filters: 0 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: use_delta_encoding: 1 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: filter_policy: bloomfilter 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: whole_key_filtering: 1 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: verify_compression: 0 2024-09-13T21:45:28.912 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: read_amp_bytes_per_bit: 0 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: format_version: 5 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: enable_index_compression: 1 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_align: 0 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: max_auto_readahead_size: 262144 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: prepopulate_block_cache: 0 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: initial_auto_readahead_size: 8192 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression: NoCompression 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T21:45:28.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.num_levels: 7 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T21:45:28.914 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T21:45:28.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T21:45:28.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T21:45:28.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.inplace_update_support: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.bloom_locality: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.max_successive_merges: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.ttl: 2592000 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T21:45:28.918 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.enable_blob_files: false 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.min_blob_size: 0 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi181/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12333, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T21:45:28.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0894210b-79e2-4480-b9d6-1ef7b7a1086f 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263928579590, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263928590786, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 2859359, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12322, "largest_seqno": 13459, "table_properties": {"data_size": 2853895, "index_size": 3203, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 11390, "raw_average_key_size": 23, "raw_value_size": 2843448, "raw_average_value_size": 5948, "num_data_blocks": 146, "num_entries": 478, "num_filter_entries": 478, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726263928, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0894210b-79e2-4480-b9d6-1ef7b7a1086f", "db_session_id": "EOJOTHJB86G8KAWDH7O9", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726263928590911, "job": 1, "event": "recovery_finished"} 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi181/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556fc533ce00 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: DB pointer 0x556fc5438000 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T21:45:28.920 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** DB Stats ** 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: L0 1/0 2.73 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T21:45:28.921 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: L6 1/0 7.60 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Sum 2/0 10.33 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 581.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 581.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T21:45:28.922 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative compaction: 0.00 GB write, 162.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval compaction: 0.00 GB write, 162.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Block cache BinnedLRUCache@0x556fc530f1f0#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T21:45:28.923 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: starting mon.smithi181 rank 1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi181 fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: mon.smithi181@-1(???) e2 preinit fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: mon.smithi181@-1(???).mds e7 new map 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: mon.smithi181@-1(???).mds e7 print_map 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: e7 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: legacy client fscid: 1 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Filesystem 'foofs' (1) 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: fs_name foofs 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: epoch 5 2024-09-13T21:45:28.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: created 2024-09-13T21:40:09.703677+0000 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: modified 2024-09-13T21:40:15.728417+0000 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: tableserver 0 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: root 0 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: session_timeout 60 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: session_autoclose 300 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: max_file_size 1099511627776 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: max_xattr_size 65536 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: required_client_features {} 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: last_failure 0 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: last_failure_osd_epoch 0 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T21:45:28.925 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: max_mds 1 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: in 0 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: up {0=14498} 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: failed 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: damaged 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: stopped 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_pools [3] 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: metadata_pool 2 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: inline_data disabled 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: balancer 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: bal_rank_mask -1 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: standby_count_wanted 1 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: qdb_cluster leader: 0 members: 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: [mds.foofs.smithi154.uynwpr{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.154:6834/3259978861,v1:172.21.15.154:6835/3259978861] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.926 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Standby daemons: 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: [mds.foofs.smithi181.mgvpzf{-1:24269} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: mon.smithi181@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: mon.smithi181@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: mon.smithi181@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: mon.smithi181@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 ceph-mon[51676]: mon.smithi181@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 bash[51657]: cf973304eaffa760c4aee5b19c947336ddf76ef6bdd7c42f0b9c21775adcd9f6 2024-09-13T21:45:28.927 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:28 smithi181 systemd[1]: Started Ceph mon.smithi181 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:45:29.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 125718 11.02 MB/sec execute 162 sec latency 44.755 ms 2024-09-13T21:45:30.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 126022 11.03 MB/sec execute 163 sec latency 57.874 ms 2024-09-13T21:45:30.897 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:45:30.897 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: mon.smithi154 calling monitor election 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: mon.smithi181 calling monitor election 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: mon.smithi154 is new leader, mons smithi154,smithi181 in quorum (ranks 0,1) 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: monmap epoch 3 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: last_changed 2024-09-13T21:45:29.723884+0000 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: created 2024-09-13T21:32:51.022089+0000 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: min_mon_release 19 (squid) 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: election_strategy: 1 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: 0: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.smithi154 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: 1: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.smithi181 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: mgrmap e42: smithi154.ayrwdi(active, since 19s), standbys: smithi181.byvkib 2024-09-13T21:45:30.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: overall HEALTH_OK 2024-09-13T21:45:30.899 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:30.899 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:30.899 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:30 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:45:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-09-13T21:45:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-09-13T21:45:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: mon.smithi154 calling monitor election 2024-09-13T21:45:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: mon.smithi181 calling monitor election 2024-09-13T21:45:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: mon.smithi154 is new leader, mons smithi154,smithi181 in quorum (ranks 0,1) 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: monmap epoch 3 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: fsid ad78d70a-7217-11ef-bceb-c7b262605968 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: last_changed 2024-09-13T21:45:29.723884+0000 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: created 2024-09-13T21:32:51.022089+0000 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: min_mon_release 19 (squid) 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: election_strategy: 1 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: 0: [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon.smithi154 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: 1: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.smithi181 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: mgrmap e42: smithi154.ayrwdi(active, since 19s), standbys: smithi181.byvkib 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: overall HEALTH_OK 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:31.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:30 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:45:31.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 126320 10.98 MB/sec execute 164 sec latency 66.612 ms 2024-09-13T21:45:31.937 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:31 smithi154 ceph-mon[65435]: pgmap v13: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 529 op/s 2024-09-13T21:45:32.003 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:31 smithi181 ceph-mon[51676]: pgmap v13: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 529 op/s 2024-09-13T21:45:32.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 126711 10.97 MB/sec execute 165 sec latency 91.325 ms 2024-09-13T21:45:33.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 126975 10.95 MB/sec execute 166 sec latency 91.832 ms 2024-09-13T21:45:33.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:33 smithi154 ceph-mon[65435]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 727 op/s 2024-09-13T21:45:33.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:33.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:33 smithi181 ceph-mon[51676]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 727 op/s 2024-09-13T21:45:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:34.893 INFO:teuthology.orchestra.run.smithi154.stdout: 5 127330 10.97 MB/sec execute 167 sec latency 61.828 ms 2024-09-13T21:45:35.277 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:35 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:35.277 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:35 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:35.277 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:35 smithi154 ceph-mon[65435]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-09-13T21:45:35.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:35 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:35.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:35 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:35.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:35 smithi181 ceph-mon[51676]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-09-13T21:45:35.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 127714 10.96 MB/sec execute 168 sec latency 91.016 ms 2024-09-13T21:45:36.896 INFO:teuthology.orchestra.run.smithi154.stdout: 5 128029 10.94 MB/sec execute 169 sec latency 86.164 ms 2024-09-13T21:45:37.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:37 smithi154 ceph-mon[65435]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-09-13T21:45:37.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:37 smithi181 ceph-mon[51676]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-09-13T21:45:37.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 128277 10.90 MB/sec execute 170 sec latency 90.159 ms 2024-09-13T21:45:38.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:38.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:38.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:38.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:45:38.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:38.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:38.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:45:38.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:45:38.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:38.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 128531 10.88 MB/sec execute 171 sec latency 59.014 ms 2024-09-13T21:45:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:45:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:39.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:45:39.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:45:39.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:39.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 128832 10.85 MB/sec execute 172 sec latency 52.354 ms 2024-09-13T21:45:39.908 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:39 smithi154 ceph-mon[65435]: Reconfiguring mon.smithi154 (monmap changed)... 2024-09-13T21:45:39.908 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:39 smithi154 ceph-mon[65435]: Reconfiguring daemon mon.smithi154 on smithi154 2024-09-13T21:45:39.908 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:39 smithi154 ceph-mon[65435]: pgmap v17: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.5 MiB/s wr, 633 op/s 2024-09-13T21:45:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:39 smithi181 ceph-mon[51676]: Reconfiguring mon.smithi154 (monmap changed)... 2024-09-13T21:45:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:39 smithi181 ceph-mon[51676]: Reconfiguring daemon mon.smithi154 on smithi154 2024-09-13T21:45:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:39 smithi181 ceph-mon[51676]: pgmap v17: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.5 MiB/s wr, 633 op/s 2024-09-13T21:45:40.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 129185 10.86 MB/sec execute 173 sec latency 37.666 ms 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: Reconfiguring mgr.smithi154.ayrwdi (monmap changed)... 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.ayrwdi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: Reconfiguring daemon mgr.smithi154.ayrwdi on smithi154 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: pgmap v18: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 401 op/s 2024-09-13T21:45:41.441 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:45:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: Reconfiguring mgr.smithi154.ayrwdi (monmap changed)... 2024-09-13T21:45:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.ayrwdi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:45:41.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:45:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: Reconfiguring daemon mgr.smithi154.ayrwdi on smithi154 2024-09-13T21:45:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: pgmap v18: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 401 op/s 2024-09-13T21:45:41.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:45:41.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 129461 10.84 MB/sec execute 174 sec latency 52.222 ms 2024-09-13T21:45:42.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 129798 10.85 MB/sec execute 175 sec latency 87.052 ms 2024-09-13T21:45:43.806 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:43 smithi154 ceph-mon[65435]: pgmap v19: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.9 MiB/s wr, 581 op/s 2024-09-13T21:45:43.806 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:43 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:43.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:43 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:43.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:43 smithi154 ceph-mon[65435]: Reconfiguring crash.smithi154 (monmap changed)... 2024-09-13T21:45:43.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:43 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:45:43.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:43 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:43.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:43 smithi154 ceph-mon[65435]: Reconfiguring daemon crash.smithi154 on smithi154 2024-09-13T21:45:43.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 129882 10.80 MB/sec execute 176 sec latency 152.517 ms 2024-09-13T21:45:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:43 smithi181 ceph-mon[51676]: pgmap v19: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.9 MiB/s wr, 581 op/s 2024-09-13T21:45:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:43 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:43 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:43 smithi181 ceph-mon[51676]: Reconfiguring crash.smithi154 (monmap changed)... 2024-09-13T21:45:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:43 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:45:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:43 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:43 smithi181 ceph-mon[51676]: Reconfiguring daemon crash.smithi154 on smithi154 2024-09-13T21:45:44.568 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:45:44.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 130053 10.74 MB/sec execute 177 sec latency 147.818 ms 2024-09-13T21:45:45.587 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:45 smithi154 ceph-mon[65435]: pgmap v20: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-09-13T21:45:45.587 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:45.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:45.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:45 smithi154 ceph-mon[65435]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T21:45:45.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:45:45.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:45.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:45 smithi154 ceph-mon[65435]: Reconfiguring daemon osd.0 on smithi154 2024-09-13T21:45:45.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:45 smithi154 ceph-mon[65435]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:45.628 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (2m) 25s ago 11m 28.8M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (11m) 25s ago 11m 6400k - 17.2.0 e1d6a67b021e f1987700146f 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (10m) 12s ago 10m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (2m) 25s ago 11m 54.8M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (5m) 25s ago 5m 36.8M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (5m) 12s ago 5m 16.7M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (114s) 25s ago 12m 474M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (63s) 12s ago 10m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (45s) 25s ago 12m 31.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:45:45.629 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (17s) 12s ago 10m 20.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (5m) 25s ago 5m 255M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (5m) 12s ago 5m 81.8M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (2m) 25s ago 11m 10.9M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (2m) 12s ago 10m 15.4M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (9m) 25s ago 9m 449M 1936M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (9m) 25s ago 9m 384M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (9m) 25s ago 9m 294M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (8m) 25s ago 8m 416M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (8m) 12s ago 8m 438M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (7m) 12s ago 7m 485M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (7m) 12s ago 7m 412M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (7m) 12s ago 7m 564M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:45:45.630 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (69s) 25s ago 10m 52.8M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:45:45.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 130425 10.73 MB/sec execute 178 sec latency 37.951 ms 2024-09-13T21:45:45.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:45 smithi181 ceph-mon[51676]: pgmap v20: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-09-13T21:45:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:45 smithi181 ceph-mon[51676]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T21:45:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:45:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:45 smithi181 ceph-mon[51676]: Reconfiguring daemon osd.0 on smithi154 2024-09-13T21:45:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:45 smithi181 ceph-mon[51676]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:46.258 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:45:46.258 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:45:46.258 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:45:46.258 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:45:46.258 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:45:46.258 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:45:46.259 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:45:46.599 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:46 smithi154 ceph-mon[65435]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:46.599 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:46 smithi154 ceph-mon[65435]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:46.600 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:46 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/270512975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:45:46.772 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:45:46.772 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:45:46.772 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:45:46.772 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:45:46.772 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:45:46.773 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:45:46.773 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr" 2024-09-13T21:45:46.773 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:45:46.773 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "4/23 daemons upgraded", 2024-09-13T21:45:46.773 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T21:45:46.773 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:45:46.773 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:45:46.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 130759 10.74 MB/sec execute 179 sec latency 47.529 ms 2024-09-13T21:45:46.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:46 smithi181 ceph-mon[51676]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:46.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:46 smithi181 ceph-mon[51676]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:46.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:46 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/270512975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:45:47.362 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:45:47.877 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: pgmap v21: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-09-13T21:45:47.877 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:47.877 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:47.878 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:47.878 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T21:45:47.878 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:45:47.878 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:47.878 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: Reconfiguring daemon osd.1 on smithi154 2024-09-13T21:45:47.878 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:47 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/3892196141' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:45:47.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 131038 10.73 MB/sec execute 180 sec latency 55.225 ms 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: pgmap v21: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: Reconfiguring daemon osd.1 on smithi154 2024-09-13T21:45:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:47 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/3892196141' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:45:48.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 131398 10.73 MB/sec execute 181 sec latency 80.629 ms 2024-09-13T21:45:49.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 131661 10.70 MB/sec execute 182 sec latency 91.746 ms 2024-09-13T21:45:49.914 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:49 smithi154 ceph-mon[65435]: pgmap v22: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 542 op/s 2024-09-13T21:45:49.914 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:49 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:49.915 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:49 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:49.915 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:49 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:45:49.915 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:49 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:49 smithi181 ceph-mon[51676]: pgmap v22: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 542 op/s 2024-09-13T21:45:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:49 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:49 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:49 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:45:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:49 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:50.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 131889 10.66 MB/sec execute 183 sec latency 47.897 ms 2024-09-13T21:45:50.972 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:50 smithi154 ceph-mon[65435]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T21:45:50.972 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:50 smithi154 ceph-mon[65435]: Reconfiguring daemon osd.2 on smithi154 2024-09-13T21:45:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:50 smithi181 ceph-mon[51676]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T21:45:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:50 smithi181 ceph-mon[51676]: Reconfiguring daemon osd.2 on smithi154 2024-09-13T21:45:51.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 132305 10.66 MB/sec execute 184 sec latency 64.734 ms 2024-09-13T21:45:51.991 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:51 smithi154 ceph-mon[65435]: pgmap v23: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.4 MiB/s wr, 338 op/s 2024-09-13T21:45:51.992 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:51.992 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:51.992 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:45:51.992 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:51 smithi181 ceph-mon[51676]: pgmap v23: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.4 MiB/s wr, 338 op/s 2024-09-13T21:45:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:45:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:52.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 132856 10.69 MB/sec execute 185 sec latency 23.481 ms 2024-09-13T21:45:52.998 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:52 smithi154 ceph-mon[65435]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T21:45:52.998 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:52 smithi154 ceph-mon[65435]: Reconfiguring daemon osd.3 on smithi154 2024-09-13T21:45:53.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:52 smithi181 ceph-mon[51676]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T21:45:53.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:52 smithi181 ceph-mon[51676]: Reconfiguring daemon osd.3 on smithi154 2024-09-13T21:45:53.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 133316 10.72 MB/sec execute 186 sec latency 38.925 ms 2024-09-13T21:45:54.018 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:53 smithi154 ceph-mon[65435]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 520 op/s 2024-09-13T21:45:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:53 smithi181 ceph-mon[51676]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 520 op/s 2024-09-13T21:45:54.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 133807 10.70 MB/sec execute 187 sec latency 32.769 ms 2024-09-13T21:45:55.028 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:54 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:55.028 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:54 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:55.028 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:54 smithi154 ceph-mon[65435]: Reconfiguring mds.foofs.smithi154.uynwpr (monmap changed)... 2024-09-13T21:45:55.028 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:54 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi154.uynwpr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:45:55.028 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:54 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:55.028 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:54 smithi154 ceph-mon[65435]: Reconfiguring daemon mds.foofs.smithi154.uynwpr on smithi154 2024-09-13T21:45:55.028 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:54 smithi154 ceph-mon[65435]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 339 op/s 2024-09-13T21:45:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:54 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:54 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:54 smithi181 ceph-mon[51676]: Reconfiguring mds.foofs.smithi154.uynwpr (monmap changed)... 2024-09-13T21:45:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:54 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi154.uynwpr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:45:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:54 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:54 smithi181 ceph-mon[51676]: Reconfiguring daemon mds.foofs.smithi154.uynwpr on smithi154 2024-09-13T21:45:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:54 smithi181 ceph-mon[51676]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 339 op/s 2024-09-13T21:45:55.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 134158 10.69 MB/sec execute 188 sec latency 34.315 ms 2024-09-13T21:45:56.076 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:45:56.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:45:56.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 134608 10.72 MB/sec execute 189 sec latency 37.981 ms 2024-09-13T21:45:57.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:57 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:57.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:57 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:57.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:57 smithi154 ceph-mon[65435]: Reconfiguring crash.smithi181 (monmap changed)... 2024-09-13T21:45:57.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:57 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:45:57.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:57 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:57.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:57 smithi154 ceph-mon[65435]: Reconfiguring daemon crash.smithi181 on smithi181 2024-09-13T21:45:57.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:57 smithi154 ceph-mon[65435]: pgmap v26: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.8 MiB/s wr, 358 op/s 2024-09-13T21:45:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:57 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:57 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:57 smithi181 ceph-mon[51676]: Reconfiguring crash.smithi181 (monmap changed)... 2024-09-13T21:45:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:57 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:45:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:57 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:57.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:57 smithi181 ceph-mon[51676]: Reconfiguring daemon crash.smithi181 on smithi181 2024-09-13T21:45:57.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:57 smithi181 ceph-mon[51676]: pgmap v26: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.8 MiB/s wr, 358 op/s 2024-09-13T21:45:57.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 135039 10.73 MB/sec execute 190 sec latency 47.606 ms 2024-09-13T21:45:58.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 135379 10.71 MB/sec execute 191 sec latency 40.936 ms 2024-09-13T21:45:59.114 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:59.114 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:59.114 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:59 smithi154 ceph-mon[65435]: Reconfiguring mgr.smithi181.byvkib (monmap changed)... 2024-09-13T21:45:59.114 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:45:59.114 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:45:59.114 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:59.114 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:59 smithi154 ceph-mon[65435]: Reconfiguring daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:45:59.114 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:45:59 smithi154 ceph-mon[65435]: pgmap v27: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.3 MiB/s wr, 617 op/s 2024-09-13T21:45:59.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:59.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:45:59.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:59 smithi181 ceph-mon[51676]: Reconfiguring mgr.smithi181.byvkib (monmap changed)... 2024-09-13T21:45:59.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.byvkib", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T21:45:59.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T21:45:59.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:45:59.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:59 smithi181 ceph-mon[51676]: Reconfiguring daemon mgr.smithi181.byvkib on smithi181 2024-09-13T21:45:59.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:45:59 smithi181 ceph-mon[51676]: pgmap v27: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.3 MiB/s wr, 617 op/s 2024-09-13T21:45:59.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 136082 10.75 MB/sec execute 192 sec latency 23.276 ms 2024-09-13T21:46:00.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:00 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:00.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:00 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:00.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:00 smithi154 ceph-mon[65435]: Reconfiguring mon.smithi181 (monmap changed)... 2024-09-13T21:46:00.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:00 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:46:00.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:00 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:46:00.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:00 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:00.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:00 smithi154 ceph-mon[65435]: Reconfiguring daemon mon.smithi181 on smithi181 2024-09-13T21:46:00.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 136812 10.83 MB/sec execute 193 sec latency 26.908 ms 2024-09-13T21:46:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:00 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:00 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:00 smithi181 ceph-mon[51676]: Reconfiguring mon.smithi181 (monmap changed)... 2024-09-13T21:46:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:00 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T21:46:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:00 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T21:46:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:00 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:00.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:00 smithi181 ceph-mon[51676]: Reconfiguring daemon mon.smithi181 on smithi181 2024-09-13T21:46:01.812 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:01 smithi181 ceph-mon[51676]: pgmap v28: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 459 op/s 2024-09-13T21:46:01.812 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:01.880 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:01 smithi154 ceph-mon[65435]: pgmap v28: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 459 op/s 2024-09-13T21:46:01.880 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:01.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:01.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:46:01.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:01.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 137602 10.84 MB/sec execute 194 sec latency 25.377 ms 2024-09-13T21:46:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:46:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:02.884 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:02 smithi154 ceph-mon[65435]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T21:46:02.885 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:02 smithi154 ceph-mon[65435]: Reconfiguring daemon osd.4 on smithi181 2024-09-13T21:46:02.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 138297 10.90 MB/sec execute 195 sec latency 50.262 ms 2024-09-13T21:46:03.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:02 smithi181 ceph-mon[51676]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T21:46:03.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:02 smithi181 ceph-mon[51676]: Reconfiguring daemon osd.4 on smithi181 2024-09-13T21:46:03.816 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:03 smithi181 ceph-mon[51676]: pgmap v29: 97 pgs: 97 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 790 op/s 2024-09-13T21:46:03.816 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:03.816 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:03.817 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:46:03.817 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:03.888 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:03 smithi154 ceph-mon[65435]: pgmap v29: 97 pgs: 97 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 790 op/s 2024-09-13T21:46:03.888 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:03.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:03.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:46:03.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:03.894 INFO:teuthology.orchestra.run.smithi154.stdout: 5 138925 10.92 MB/sec execute 196 sec latency 26.557 ms 2024-09-13T21:46:04.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 139729 10.97 MB/sec execute 197 sec latency 21.035 ms 2024-09-13T21:46:04.895 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:04 smithi154 ceph-mon[65435]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T21:46:04.895 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:04 smithi154 ceph-mon[65435]: Reconfiguring daemon osd.5 on smithi181 2024-09-13T21:46:05.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:04 smithi181 ceph-mon[51676]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T21:46:05.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:04 smithi181 ceph-mon[51676]: Reconfiguring daemon osd.5 on smithi181 2024-09-13T21:46:05.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 140430 11.04 MB/sec execute 198 sec latency 29.170 ms 2024-09-13T21:46:05.909 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:05 smithi154 ceph-mon[65435]: pgmap v30: 97 pgs: 97 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 608 op/s 2024-09-13T21:46:05.909 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:05 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:05.910 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:05 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:05.910 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:05 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:46:05.910 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:05 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:06.006 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:05 smithi181 ceph-mon[51676]: pgmap v30: 97 pgs: 97 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 608 op/s 2024-09-13T21:46:06.006 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:05 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:06.006 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:05 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:06.007 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:05 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:46:06.007 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:05 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:06.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 141174 11.05 MB/sec execute 199 sec latency 33.261 ms 2024-09-13T21:46:06.915 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:06 smithi154 ceph-mon[65435]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T21:46:06.915 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:06 smithi154 ceph-mon[65435]: Reconfiguring daemon osd.6 on smithi181 2024-09-13T21:46:07.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:06 smithi181 ceph-mon[51676]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T21:46:07.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:06 smithi181 ceph-mon[51676]: Reconfiguring daemon osd.6 on smithi181 2024-09-13T21:46:07.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 141788 11.10 MB/sec execute 200 sec latency 39.540 ms 2024-09-13T21:46:07.920 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:07 smithi154 ceph-mon[65435]: pgmap v31: 97 pgs: 97 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 710 op/s 2024-09-13T21:46:07.920 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:07.920 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:07.920 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:46:07.921 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:07 smithi181 ceph-mon[51676]: pgmap v31: 97 pgs: 97 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 710 op/s 2024-09-13T21:46:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:46:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:08.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 142363 11.11 MB/sec execute 201 sec latency 32.702 ms 2024-09-13T21:46:08.939 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:08 smithi154 ceph-mon[65435]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T21:46:08.939 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:08 smithi154 ceph-mon[65435]: Reconfiguring daemon osd.7 on smithi181 2024-09-13T21:46:08.939 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:08 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:08.939 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:08 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:08.939 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:08 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.mgvpzf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:46:08.940 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:08 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:09.096 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:08 smithi181 ceph-mon[51676]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T21:46:09.096 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:08 smithi181 ceph-mon[51676]: Reconfiguring daemon osd.7 on smithi181 2024-09-13T21:46:09.096 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:08 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:09.096 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:08 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:09.096 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:08 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.mgvpzf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:46:09.097 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:08 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:09.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 142570 11.08 MB/sec execute 202 sec latency 60.287 ms 2024-09-13T21:46:10.003 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:09 smithi154 ceph-mon[65435]: pgmap v32: 97 pgs: 97 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 10 MiB/s wr, 995 op/s 2024-09-13T21:46:10.003 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:09 smithi154 ceph-mon[65435]: Reconfiguring mds.foofs.smithi181.mgvpzf (monmap changed)... 2024-09-13T21:46:10.004 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:09 smithi154 ceph-mon[65435]: Reconfiguring daemon mds.foofs.smithi181.mgvpzf on smithi181 2024-09-13T21:46:10.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:09 smithi181 ceph-mon[51676]: pgmap v32: 97 pgs: 97 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 10 MiB/s wr, 995 op/s 2024-09-13T21:46:10.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:09 smithi181 ceph-mon[51676]: Reconfiguring mds.foofs.smithi181.mgvpzf (monmap changed)... 2024-09-13T21:46:10.126 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:09 smithi181 ceph-mon[51676]: Reconfiguring daemon mds.foofs.smithi181.mgvpzf on smithi181 2024-09-13T21:46:10.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 142886 11.09 MB/sec execute 203 sec latency 67.702 ms 2024-09-13T21:46:11.284 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all mon 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi154"}]: dispatch 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi154"}]': finished 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi181"}]: dispatch 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi181"}]': finished 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: pgmap v33: 97 pgs: 97 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.6 MiB/s wr, 736 op/s 2024-09-13T21:46:11.285 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all mon 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi154"}]: dispatch 2024-09-13T21:46:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi154"}]': finished 2024-09-13T21:46:11.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi181"}]: dispatch 2024-09-13T21:46:11.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi181"}]': finished 2024-09-13T21:46:11.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: pgmap v33: 97 pgs: 97 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.6 MiB/s wr, 736 op/s 2024-09-13T21:46:11.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:46:11.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 143263 11.06 MB/sec execute 204 sec latency 75.978 ms 2024-09-13T21:46:12.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 143710 11.08 MB/sec execute 205 sec latency 39.647 ms 2024-09-13T21:46:13.523 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:13 smithi154 ceph-mon[65435]: Upgrade: Updating crash.smithi154 (1/2) 2024-09-13T21:46:13.523 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:13.523 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:46:13.523 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:13.523 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:13 smithi154 ceph-mon[65435]: Deploying daemon crash.smithi154 on smithi154 2024-09-13T21:46:13.523 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:13 smithi154 ceph-mon[65435]: pgmap v34: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-09-13T21:46:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:13 smithi181 ceph-mon[51676]: Upgrade: Updating crash.smithi154 (1/2) 2024-09-13T21:46:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:46:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:13 smithi181 ceph-mon[51676]: Deploying daemon crash.smithi154 on smithi154 2024-09-13T21:46:13.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:13 smithi181 ceph-mon[51676]: pgmap v34: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-09-13T21:46:13.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 144126 11.09 MB/sec execute 206 sec latency 57.759 ms 2024-09-13T21:46:14.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 144593 11.07 MB/sec execute 207 sec latency 33.731 ms 2024-09-13T21:46:15.595 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:15 smithi154 ceph-mon[65435]: pgmap v35: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 675 op/s 2024-09-13T21:46:15.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 144904 11.06 MB/sec execute 208 sec latency 35.153 ms 2024-09-13T21:46:15.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:15 smithi181 ceph-mon[51676]: pgmap v35: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 675 op/s 2024-09-13T21:46:16.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 145576 11.11 MB/sec execute 209 sec latency 43.788 ms 2024-09-13T21:46:17.508 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:17 smithi154 ceph-mon[65435]: pgmap v36: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 814 op/s 2024-09-13T21:46:17.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 146013 11.10 MB/sec execute 210 sec latency 39.148 ms 2024-09-13T21:46:17.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:17 smithi181 ceph-mon[51676]: pgmap v36: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 8.8 MiB/s wr, 814 op/s 2024-09-13T21:46:17.992 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:46:18.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 146368 11.10 MB/sec execute 211 sec latency 39.887 ms 2024-09-13T21:46:19.126 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:46:19.126 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (3m) 59s ago 11m 28.8M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 starting - - - - 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (11m) 45s ago 11m 6405k - 17.2.0 e1d6a67b021e 735977650ee8 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (3m) 59s ago 11m 54.8M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (6m) 59s ago 6m 36.8M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (6m) 45s ago 6m 16.7M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (2m) 59s ago 13m 474M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (96s) 45s ago 11m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (79s) 59s ago 13m 31.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (51s) 45s ago 11m 20.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (5m) 59s ago 5m 255M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (5m) 45s ago 5m 81.8M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (3m) 59s ago 11m 10.9M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:46:19.127 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (2m) 45s ago 11m 15.4M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (10m) 59s ago 10m 449M 1936M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (10m) 59s ago 10m 384M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (9m) 59s ago 9m 294M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (9m) 59s ago 9m 416M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (8m) 45s ago 8m 438M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (8m) 45s ago 8m 485M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (7m) 45s ago 7m 412M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (7m) 45s ago 7m 564M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:46:19.128 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (103s) 59s ago 10m 52.8M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:46:19.810 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:46:19.811 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:46:19.815 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:19 smithi154 ceph-mon[65435]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:19.816 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:19 smithi154 ceph-mon[65435]: pgmap v37: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 8.9 MiB/s wr, 833 op/s 2024-09-13T21:46:19.816 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:19 smithi154 ceph-mon[65435]: from='client.44119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:19.816 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:19.816 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:19.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 146881 11.10 MB/sec execute 212 sec latency 43.624 ms 2024-09-13T21:46:20.084 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:19 smithi181 ceph-mon[51676]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:20.084 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:19 smithi181 ceph-mon[51676]: pgmap v37: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 8.9 MiB/s wr, 833 op/s 2024-09-13T21:46:20.084 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:19 smithi181 ceph-mon[51676]: from='client.44119 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:20.084 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:20.085 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:20.366 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:46:20.366 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:46:20.366 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:46:20.366 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:46:20.366 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:46:20.366 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:46:20.367 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr" 2024-09-13T21:46:20.367 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:46:20.367 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "4/23 daemons upgraded", 2024-09-13T21:46:20.367 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading crash daemons", 2024-09-13T21:46:20.367 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:46:20.367 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:46:20.834 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:20 smithi154 ceph-mon[65435]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:20.835 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:20 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/221336105' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:20.835 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:20.835 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:46:20.835 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:20.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:20 smithi181 ceph-mon[51676]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:20.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:20 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/221336105' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:20.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:20 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:20.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:20 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T21:46:20.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:20 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:20.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 147252 11.11 MB/sec execute 213 sec latency 46.332 ms 2024-09-13T21:46:21.006 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:46:21.860 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:21 smithi154 ceph-mon[65435]: Upgrade: Updating crash.smithi181 (2/2) 2024-09-13T21:46:21.860 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:21 smithi154 ceph-mon[65435]: Deploying daemon crash.smithi181 on smithi181 2024-09-13T21:46:21.861 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:21 smithi154 ceph-mon[65435]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:21.861 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:21 smithi154 ceph-mon[65435]: pgmap v38: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.5 MiB/s wr, 530 op/s 2024-09-13T21:46:21.861 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:21 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/700597969' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:46:21.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 147775 11.13 MB/sec execute 214 sec latency 33.848 ms 2024-09-13T21:46:22.035 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:21 smithi181 ceph-mon[51676]: Upgrade: Updating crash.smithi181 (2/2) 2024-09-13T21:46:22.035 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:21 smithi181 ceph-mon[51676]: Deploying daemon crash.smithi181 on smithi181 2024-09-13T21:46:22.035 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:21 smithi181 ceph-mon[51676]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:22.035 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:21 smithi181 ceph-mon[51676]: pgmap v38: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.5 MiB/s wr, 530 op/s 2024-09-13T21:46:22.035 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:21 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/700597969' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:46:22.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 148221 11.11 MB/sec execute 215 sec latency 32.540 ms 2024-09-13T21:46:23.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 148503 11.10 MB/sec execute 216 sec latency 41.409 ms 2024-09-13T21:46:23.908 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:23 smithi154 ceph-mon[65435]: pgmap v39: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 805 op/s 2024-09-13T21:46:23.928 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:23 smithi181 ceph-mon[51676]: pgmap v39: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 805 op/s 2024-09-13T21:46:24.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 148876 11.11 MB/sec execute 217 sec latency 47.137 ms 2024-09-13T21:46:25.221 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:25.221 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:25.221 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:25.222 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:25 smithi154 ceph-mon[65435]: pgmap v40: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 535 op/s 2024-09-13T21:46:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:25.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:25 smithi181 ceph-mon[51676]: pgmap v40: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 535 op/s 2024-09-13T21:46:25.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 149282 11.11 MB/sec execute 218 sec latency 49.247 ms 2024-09-13T21:46:26.231 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:26 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:46:26.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:26 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:46:26.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 149802 11.13 MB/sec execute 219 sec latency 52.265 ms 2024-09-13T21:46:27.236 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:27 smithi154 ceph-mon[65435]: pgmap v41: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 730 op/s 2024-09-13T21:46:27.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:27 smithi181 ceph-mon[51676]: pgmap v41: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 730 op/s 2024-09-13T21:46:27.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 150585 11.16 MB/sec execute 220 sec latency 32.113 ms 2024-09-13T21:46:28.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 151054 11.19 MB/sec execute 221 sec latency 39.681 ms 2024-09-13T21:46:28.997 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:28 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:28.997 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:28 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:28.997 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:28 smithi154 ceph-mon[65435]: pgmap v42: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 630 op/s 2024-09-13T21:46:28.998 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:28 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:28.998 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:28 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:29.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:28 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:29.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:28 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:29.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:28 smithi181 ceph-mon[51676]: pgmap v42: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 630 op/s 2024-09-13T21:46:29.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:28 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:29.261 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:28 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:29.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 151586 11.17 MB/sec execute 222 sec latency 75.722 ms 2024-09-13T21:46:30.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 151918 11.16 MB/sec execute 223 sec latency 48.287 ms 2024-09-13T21:46:31.537 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:31 smithi154 ceph-mon[65435]: pgmap v43: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 509 op/s 2024-09-13T21:46:31.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:31 smithi181 ceph-mon[51676]: pgmap v43: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 509 op/s 2024-09-13T21:46:31.895 INFO:teuthology.orchestra.run.smithi154.stdout: 5 152380 11.19 MB/sec execute 224 sec latency 33.497 ms 2024-09-13T21:46:32.896 INFO:teuthology.orchestra.run.smithi154.stdout: 5 152886 11.19 MB/sec execute 225 sec latency 60.526 ms 2024-09-13T21:46:33.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:33.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:33.322 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:33 smithi154 ceph-mon[65435]: pgmap v44: 97 pgs: 97 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 835 op/s 2024-09-13T21:46:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:33.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:33 smithi181 ceph-mon[51676]: pgmap v44: 97 pgs: 97 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 835 op/s 2024-09-13T21:46:33.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 153196 11.17 MB/sec execute 226 sec latency 62.082 ms 2024-09-13T21:46:34.899 INFO:teuthology.orchestra.run.smithi154.stdout: 5 153563 11.18 MB/sec execute 227 sec latency 75.842 ms 2024-09-13T21:46:35.100 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:35 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:35.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:35 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:35.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:35 smithi154 ceph-mon[65435]: pgmap v45: 97 pgs: 97 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 559 op/s 2024-09-13T21:46:35.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:35 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:35.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:35 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:35.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:35 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:35.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:35 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:35.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:35 smithi181 ceph-mon[51676]: pgmap v45: 97 pgs: 97 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 559 op/s 2024-09-13T21:46:35.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:35 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:35.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:35 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:35.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 153877 11.15 MB/sec execute 228 sec latency 49.056 ms 2024-09-13T21:46:36.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 154136 11.14 MB/sec execute 229 sec latency 74.281 ms 2024-09-13T21:46:37.683 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:37 smithi154 ceph-mon[65435]: pgmap v46: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.1 MiB/s wr, 707 op/s 2024-09-13T21:46:37.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 154370 11.13 MB/sec execute 230 sec latency 60.015 ms 2024-09-13T21:46:37.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:37 smithi181 ceph-mon[51676]: pgmap v46: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.1 MiB/s wr, 707 op/s 2024-09-13T21:46:38.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 154738 11.13 MB/sec execute 231 sec latency 45.945 ms 2024-09-13T21:46:39.598 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:39 smithi154 ceph-mon[65435]: pgmap v47: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 583 op/s 2024-09-13T21:46:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:39 smithi181 ceph-mon[51676]: pgmap v47: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 583 op/s 2024-09-13T21:46:39.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 155324 11.12 MB/sec execute 232 sec latency 49.539 ms 2024-09-13T21:46:40.798 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:40 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:46:40.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:40 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:46:40.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 155858 11.15 MB/sec execute 233 sec latency 103.903 ms 2024-09-13T21:46:41.552 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: pgmap v48: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.6 MiB/s wr, 544 op/s 2024-09-13T21:46:41.552 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.552 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.552 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:41.552 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:46:41.552 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi154"}]: dispatch 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi154"}]': finished 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi181"}]: dispatch 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi181"}]': finished 2024-09-13T21:46:41.553 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T21:46:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: pgmap v48: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.6 MiB/s wr, 544 op/s 2024-09-13T21:46:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi154"}]: dispatch 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi154"}]': finished 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi181"}]: dispatch 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi181"}]': finished 2024-09-13T21:46:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T21:46:41.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 156522 11.18 MB/sec execute 234 sec latency 38.270 ms 2024-09-13T21:46:42.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:42 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all crash 2024-09-13T21:46:42.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:42 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T21:46:42.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:42 smithi154 ceph-mon[65435]: Upgrade: osd.0 is safe to restart 2024-09-13T21:46:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:42 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all crash 2024-09-13T21:46:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:42 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T21:46:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:42 smithi181 ceph-mon[51676]: Upgrade: osd.0 is safe to restart 2024-09-13T21:46:42.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 157075 11.19 MB/sec execute 235 sec latency 39.449 ms 2024-09-13T21:46:43.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:43 smithi154 ceph-mon[65435]: pgmap v49: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 803 op/s 2024-09-13T21:46:43.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:43 smithi154 ceph-mon[65435]: mgrmap e43: smithi154.ayrwdi(active, since 92s), standbys: smithi181.byvkib 2024-09-13T21:46:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:43 smithi181 ceph-mon[51676]: pgmap v49: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 803 op/s 2024-09-13T21:46:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:43 smithi181 ceph-mon[51676]: mgrmap e43: smithi154.ayrwdi(active, since 92s), standbys: smithi181.byvkib 2024-09-13T21:46:43.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 157474 11.18 MB/sec execute 236 sec latency 52.993 ms 2024-09-13T21:46:44.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:44 smithi154 ceph-mon[65435]: Upgrade: Updating osd.0 2024-09-13T21:46:44.790 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:44 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:44.791 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:44 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:46:44.791 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:44 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:44.791 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:44 smithi154 ceph-mon[65435]: Deploying daemon osd.0 on smithi154 2024-09-13T21:46:44.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 157729 11.17 MB/sec execute 237 sec latency 62.742 ms 2024-09-13T21:46:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:44 smithi181 ceph-mon[51676]: Upgrade: Updating osd.0 2024-09-13T21:46:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:44 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:44 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T21:46:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:44 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:46:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:44 smithi181 ceph-mon[51676]: Deploying daemon osd.0 on smithi154 2024-09-13T21:46:45.816 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:45 smithi154 ceph-mon[65435]: pgmap v50: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-09-13T21:46:45.900 INFO:teuthology.orchestra.run.smithi154.stdout: 5 158175 11.20 MB/sec execute 238 sec latency 52.419 ms 2024-09-13T21:46:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:45 smithi181 ceph-mon[51676]: pgmap v50: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-09-13T21:46:46.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 158675 11.18 MB/sec execute 239 sec latency 35.059 ms 2024-09-13T21:46:47.847 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:47 smithi154 ceph-mon[65435]: pgmap v51: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 551 op/s 2024-09-13T21:46:47.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 158950 11.16 MB/sec execute 240 sec latency 75.206 ms 2024-09-13T21:46:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:47 smithi181 ceph-mon[51676]: pgmap v51: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 551 op/s 2024-09-13T21:46:48.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 159321 11.17 MB/sec execute 241 sec latency 63.494 ms 2024-09-13T21:46:49.130 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:48 smithi154 systemd[1]: Stopping Ceph osd.0 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:46:49.380 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:49 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[34142]: 2024-09-13T21:46:49.363+0000 7f86f7e1a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:46:49.381 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:49 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[34142]: 2024-09-13T21:46:49.363+0000 7f86f7e1a700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-13T21:46:49.381 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:49 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[34142]: 2024-09-13T21:46:49.363+0000 7f86f7e1a700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:46:49.896 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:49 smithi154 ceph-mon[65435]: pgmap v52: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 609 op/s 2024-09-13T21:46:49.897 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:49 smithi154 ceph-mon[65435]: osd.0 marked itself down and dead 2024-09-13T21:46:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:49 smithi181 ceph-mon[51676]: pgmap v52: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 609 op/s 2024-09-13T21:46:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:49 smithi181 ceph-mon[51676]: osd.0 marked itself down and dead 2024-09-13T21:46:49.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 159720 11.17 MB/sec execute 242 sec latency 119.365 ms 2024-09-13T21:46:50.779 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:50 smithi154 podman[70879]: 2024-09-13 21:46:50.388018213 +0000 UTC m=+1.160264735 container died 03f97205e5ea2cfac91df1e6602e33e3ed23d236de0a5de2b5a0698ee3e6f546 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-type=git) 2024-09-13T21:46:50.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 159768 11.13 MB/sec execute 243 sec latency 1022.227 ms 2024-09-13T21:46:51.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:50 smithi154 ceph-mon[65435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:46:51.030 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:50 smithi154 ceph-mon[65435]: osdmap e73: 8 total, 7 up, 8 in 2024-09-13T21:46:51.031 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:50 smithi154 ceph-mon[65435]: osdmap e74: 8 total, 7 up, 8 in 2024-09-13T21:46:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:50 smithi181 ceph-mon[51676]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:46:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:50 smithi181 ceph-mon[51676]: osdmap e73: 8 total, 7 up, 8 in 2024-09-13T21:46:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:50 smithi181 ceph-mon[51676]: osdmap e74: 8 total, 7 up, 8 in 2024-09-13T21:46:51.285 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:51 smithi154 podman[70879]: 2024-09-13 21:46:51.101951009 +0000 UTC m=+1.874197537 container cleanup 03f97205e5ea2cfac91df1e6602e33e3ed23d236de0a5de2b5a0698ee3e6f546 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, release=754, ceph=True, name=centos-stream, version=8, GIT_CLEAN=True) 2024-09-13T21:46:51.285 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:51 smithi154 bash[70879]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0 2024-09-13T21:46:51.538 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:51 smithi154 podman[70894]: 2024-09-13 21:46:51.41859004 +0000 UTC m=+1.026152599 container remove 03f97205e5ea2cfac91df1e6602e33e3ed23d236de0a5de2b5a0698ee3e6f546 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, version=8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , ceph=True, architecture=x86_64, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-13T21:46:51.641 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:46:51.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:51 smithi154 ceph-mon[65435]: pgmap v54: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.5 MiB/s wr, 645 op/s 2024-09-13T21:46:51.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 160113 11.11 MB/sec execute 244 sec latency 54.407 ms 2024-09-13T21:46:52.047 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:52 smithi154 podman[71025]: 2024-09-13 21:46:51.90871973 +0000 UTC m=+0.023035498 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:46:52.047 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:52 smithi154 podman[71025]: 2024-09-13 21:46:52.035900612 +0000 UTC m=+0.150216369 container create ffe9c98eebc6ab4df1c35d0b89c49d01f0c35bcecd9b919e1c103f063e12c1e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:46:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:51 smithi181 ceph-mon[51676]: pgmap v54: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.5 MiB/s wr, 645 op/s 2024-09-13T21:46:52.478 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:46:52.478 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (3m) 18s ago 12m 27.1M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:46:52.478 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (34s) 18s ago 12m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:46:52.478 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (28s) 24s ago 11m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:46:52.478 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (3m) 18s ago 12m 52.7M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:46:52.478 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (6m) 18s ago 6m 36.6M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:46:52.478 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (6m) 24s ago 6m 16.9M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:46:52.478 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (3m) 18s ago 13m 494M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (2m) 24s ago 11m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (112s) 18s ago 13m 43.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (84s) 24s ago 11m 36.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (6m) 18s ago 6m 260M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (6m) 24s ago 6m 81.9M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (3m) 18s ago 12m 15.6M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (3m) 24s ago 11m 15.1M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (11m) 18s ago 11m 534M 1936M 17.2.0 e1d6a67b021e 03f97205e5ea 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (10m) 18s ago 10m 452M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (10m) 18s ago 10m 355M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:46:52.479 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (9m) 18s ago 9m 491M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:46:52.480 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (9m) 24s ago 9m 489M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:46:52.480 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (8m) 24s ago 8m 310M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:46:52.480 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (8m) 24s ago 8m 491M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:46:52.480 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (8m) 24s ago 8m 374M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:46:52.480 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (2m) 18s ago 11m 62.2M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:46:52.554 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:52 smithi154 podman[71025]: 2024-09-13 21:46:52.43561316 +0000 UTC m=+0.549928927 container init ffe9c98eebc6ab4df1c35d0b89c49d01f0c35bcecd9b919e1c103f063e12c1e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T21:46:52.555 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:52 smithi154 podman[71025]: 2024-09-13 21:46:52.443911766 +0000 UTC m=+0.558227526 container start ffe9c98eebc6ab4df1c35d0b89c49d01f0c35bcecd9b919e1c103f063e12c1e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True) 2024-09-13T21:46:52.555 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:52 smithi154 podman[71025]: 2024-09-13 21:46:52.532211385 +0000 UTC m=+0.646527151 container attach ffe9c98eebc6ab4df1c35d0b89c49d01f0c35bcecd9b919e1c103f063e12c1e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T21:46:52.806 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:52 smithi154 ceph-mon[65435]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:52.806 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:52 smithi154 podman[71025]: 2024-09-13 21:46:52.761761397 +0000 UTC m=+0.876077154 container died ffe9c98eebc6ab4df1c35d0b89c49d01f0c35bcecd9b919e1c103f063e12c1e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be) 2024-09-13T21:46:52.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 160401 11.11 MB/sec execute 245 sec latency 44.394 ms 2024-09-13T21:46:52.972 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:46:52.972 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:46:52.973 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-13T21:46:52.974 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T21:46:52.974 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:46:52.974 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:46:53.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:52 smithi181 ceph-mon[51676]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:53.417 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "6/23 daemons upgraded", 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:46:53.418 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:46:53.419 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:46:53.817 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:53 smithi154 ceph-mon[65435]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:53.817 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:53 smithi154 ceph-mon[65435]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 692 op/s; 218/1623 objects degraded (13.432%) 2024-09-13T21:46:53.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:53 smithi154 ceph-mon[65435]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:53.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:53 smithi154 ceph-mon[65435]: Health check failed: Degraded data redundancy: 218/1623 objects degraded (13.432%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:46:53.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:53 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/1532793976' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:53.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 160924 11.11 MB/sec execute 246 sec latency 38.893 ms 2024-09-13T21:46:54.069 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:53 smithi154 podman[71092]: 2024-09-13 21:46:53.864962911 +0000 UTC m=+1.096401200 container remove ffe9c98eebc6ab4df1c35d0b89c49d01f0c35bcecd9b919e1c103f063e12c1e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be) 2024-09-13T21:46:54.069 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:53 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T21:46:54.069 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:53 smithi154 systemd[1]: Stopped Ceph osd.0 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:46:54.070 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:53 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 42.215s CPU time. 2024-09-13T21:46:54.116 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 218/1623 objects degraded (13.432%), 29 pgs degraded 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: osd.0 (root=default,host=smithi154) is down 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 218/1623 objects degraded (13.432%), 29 pgs degraded 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.5 is active+undersized+degraded, acting [2,4] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-13T21:46:54.117 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-13T21:46:54.118 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-13T21:46:54.119 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-13T21:46:54.119 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-13T21:46:54.119 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-13T21:46:54.119 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-13T21:46:54.119 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-13T21:46:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:53 smithi181 ceph-mon[51676]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:53 smithi181 ceph-mon[51676]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 692 op/s; 218/1623 objects degraded (13.432%) 2024-09-13T21:46:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:53 smithi181 ceph-mon[51676]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:53 smithi181 ceph-mon[51676]: Health check failed: Degraded data redundancy: 218/1623 objects degraded (13.432%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:46:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:53 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/1532793976' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:46:54.319 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:54 smithi154 systemd[1]: Starting Ceph osd.0 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:46:54.570 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:54 smithi154 podman[71230]: 2024-09-13 21:46:54.456604774 +0000 UTC m=+0.022315746 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:46:54.823 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:54 smithi154 ceph-mon[65435]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:54.823 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:54 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2512150079' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:46:54.823 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:54 smithi154 podman[71230]: 2024-09-13 21:46:54.591055555 +0000 UTC m=+0.156766519 container create c8d5301d779e406e206807be87d32e06af4f81a51cd852fd68cd12c6e8d28dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T21:46:54.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 161494 11.14 MB/sec execute 247 sec latency 28.676 ms 2024-09-13T21:46:55.073 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:54 smithi154 podman[71230]: 2024-09-13 21:46:54.996294316 +0000 UTC m=+0.562005290 container init c8d5301d779e406e206807be87d32e06af4f81a51cd852fd68cd12c6e8d28dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T21:46:55.073 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:55 smithi154 podman[71230]: 2024-09-13 21:46:55.008897719 +0000 UTC m=+0.574608687 container start c8d5301d779e406e206807be87d32e06af4f81a51cd852fd68cd12c6e8d28dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T21:46:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:54 smithi181 ceph-mon[51676]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:46:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:54 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2512150079' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:46:55.278 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:55 smithi154 podman[71230]: 2024-09-13 21:46:55.129458938 +0000 UTC m=+0.695169910 container attach c8d5301d779e406e206807be87d32e06af4f81a51cd852fd68cd12c6e8d28dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be) 2024-09-13T21:46:55.278 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:55 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:46:55.278 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:55 smithi154 bash[71230]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:46:55.278 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:55 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:46:55.278 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:55 smithi154 bash[71230]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:46:55.826 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:55 smithi154 ceph-mon[65435]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.8 MiB/s wr, 582 op/s; 218/1623 objects degraded (13.432%) 2024-09-13T21:46:55.827 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:55.827 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:46:55.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 162265 11.17 MB/sec execute 248 sec latency 50.356 ms 2024-09-13T21:46:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:55 smithi181 ceph-mon[51676]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.8 MiB/s wr, 582 op/s; 218/1623 objects degraded (13.432%) 2024-09-13T21:46:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:46:56.830 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:56 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T21:46:56.831 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:56 smithi154 bash[71230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T21:46:56.831 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:56 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T21:46:56.831 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:56 smithi154 bash[71230]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T21:46:56.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 163028 11.22 MB/sec execute 249 sec latency 23.174 ms 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 bash[71230]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 bash[71230]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 bash[71230]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 bash[71230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate[71244]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 bash[71230]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T21:46:57.336 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:57 smithi154 podman[71230]: 2024-09-13 21:46:57.207268793 +0000 UTC m=+2.772979773 container died c8d5301d779e406e206807be87d32e06af4f81a51cd852fd68cd12c6e8d28dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T21:46:57.841 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:57 smithi154 ceph-mon[65435]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.4 MiB/s wr, 732 op/s; 237/1764 objects degraded (13.435%) 2024-09-13T21:46:57.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 163734 11.24 MB/sec execute 250 sec latency 24.086 ms 2024-09-13T21:46:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:57 smithi181 ceph-mon[51676]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.4 MiB/s wr, 732 op/s; 237/1764 objects degraded (13.435%) 2024-09-13T21:46:58.344 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:58 smithi154 podman[71383]: 2024-09-13 21:46:58.154844871 +0000 UTC m=+0.940192295 container remove c8d5301d779e406e206807be87d32e06af4f81a51cd852fd68cd12c6e8d28dbf (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:46:58.599 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:58 smithi154 podman[71440]: 2024-09-13 21:46:58.324281002 +0000 UTC m=+0.026613692 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:46:58.599 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:58 smithi154 podman[71440]: 2024-09-13 21:46:58.480967707 +0000 UTC m=+0.183300400 container create 6e650440345fd0fe0359c1aa262f74277108550b523d39c47ab1a3e2a3be10f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T21:46:58.854 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:58 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 245/1806 objects degraded (13.566%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:46:58.854 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:58 smithi154 podman[71440]: 2024-09-13 21:46:58.801384346 +0000 UTC m=+0.503717037 container init 6e650440345fd0fe0359c1aa262f74277108550b523d39c47ab1a3e2a3be10f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be) 2024-09-13T21:46:58.854 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:58 smithi154 podman[71440]: 2024-09-13 21:46:58.808427789 +0000 UTC m=+0.510760471 container start 6e650440345fd0fe0359c1aa262f74277108550b523d39c47ab1a3e2a3be10f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-13T21:46:58.854 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:58 smithi154 ceph-osd[71458]: -- 172.21.15.154:0/1616894217 <== mon.0 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d7eeba61e0 con 0x55d7eeb80000 2024-09-13T21:46:58.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 164131 11.24 MB/sec execute 251 sec latency 48.188 ms 2024-09-13T21:46:59.104 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:58 smithi154 bash[71440]: 6e650440345fd0fe0359c1aa262f74277108550b523d39c47ab1a3e2a3be10f6 2024-09-13T21:46:59.104 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:46:58 smithi154 systemd[1]: Started Ceph osd.0 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:46:59.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:58 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 245/1806 objects degraded (13.566%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:46:59.880 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:59 smithi154 ceph-mon[65435]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 697 op/s; 245/1806 objects degraded (13.566%) 2024-09-13T21:46:59.880 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:59.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:46:59.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:46:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:46:59.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 164519 11.23 MB/sec execute 252 sec latency 61.194 ms 2024-09-13T21:47:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:59 smithi181 ceph-mon[51676]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 697 op/s; 245/1806 objects degraded (13.566%) 2024-09-13T21:47:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:46:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:00.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 164870 11.23 MB/sec execute 253 sec latency 44.192 ms 2024-09-13T21:47:01.176 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:47:01 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[71454]: 2024-09-13T21:47:01.155+0000 7f1dae079640 -1 Falling back to public interface 2024-09-13T21:47:01.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 165225 11.23 MB/sec execute 254 sec latency 47.868 ms 2024-09-13T21:47:01.935 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:01 smithi154 ceph-mon[65435]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.2 MiB/s wr, 604 op/s; 245/1806 objects degraded (13.566%) 2024-09-13T21:47:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:01 smithi181 ceph-mon[51676]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.2 MiB/s wr, 604 op/s; 245/1806 objects degraded (13.566%) 2024-09-13T21:47:02.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 165619 11.22 MB/sec execute 255 sec latency 43.174 ms 2024-09-13T21:47:03.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 165915 11.20 MB/sec execute 256 sec latency 48.092 ms 2024-09-13T21:47:03.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:03 smithi154 ceph-mon[65435]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 816 op/s; 212/1686 objects degraded (12.574%) 2024-09-13T21:47:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:03 smithi181 ceph-mon[51676]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 816 op/s; 212/1686 objects degraded (12.574%) 2024-09-13T21:47:04.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 166171 11.19 MB/sec execute 257 sec latency 46.317 ms 2024-09-13T21:47:04.990 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:04 smithi154 ceph-mon[65435]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.4 MiB/s wr, 611 op/s; 212/1686 objects degraded (12.574%) 2024-09-13T21:47:05.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:04 smithi181 ceph-mon[51676]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.4 MiB/s wr, 611 op/s; 212/1686 objects degraded (12.574%) 2024-09-13T21:47:05.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 166519 11.19 MB/sec execute 258 sec latency 50.285 ms 2024-09-13T21:47:06.018 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:05 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 212/1686 objects degraded (12.574%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:06.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:05 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 212/1686 objects degraded (12.574%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:06.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 166920 11.18 MB/sec execute 259 sec latency 51.643 ms 2024-09-13T21:47:07.027 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:06 smithi154 ceph-mon[65435]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 651 op/s; 210/1674 objects degraded (12.545%) 2024-09-13T21:47:07.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:06 smithi181 ceph-mon[51676]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 651 op/s; 210/1674 objects degraded (12.545%) 2024-09-13T21:47:07.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 167248 11.17 MB/sec execute 260 sec latency 46.240 ms 2024-09-13T21:47:08.587 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:08 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:08.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:08 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:08.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:08 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:08.588 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:08 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:08.588 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:47:08 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[71454]: 2024-09-13T21:47:08.553+0000 7f1dae079640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-13T21:47:08.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:08 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:08.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:08 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:08.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:08 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:08.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:08 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:08.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 167421 11.14 MB/sec execute 261 sec latency 46.908 ms 2024-09-13T21:47:09.594 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:09 smithi154 ceph-mon[65435]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 508 op/s; 206/1629 objects degraded (12.646%) 2024-09-13T21:47:09.655 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:47:09 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[71454]: 2024-09-13T21:47:09.644+0000 7f1dae079640 -1 osd.0 72 log_to_monitors true 2024-09-13T21:47:09.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:09 smithi181 ceph-mon[51676]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 508 op/s; 206/1629 objects degraded (12.646%) 2024-09-13T21:47:09.901 INFO:teuthology.orchestra.run.smithi154.stdout: 5 167791 11.15 MB/sec execute 262 sec latency 52.679 ms 2024-09-13T21:47:10.645 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:10 smithi154 ceph-mon[65435]: from='osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:47:10.645 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:47:10.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:10 smithi181 ceph-mon[51676]: from='osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T21:47:10.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:47:10.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 168055 11.12 MB/sec execute 263 sec latency 97.152 ms 2024-09-13T21:47:11.165 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:47:11 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[71454]: 2024-09-13T21:47:11.052+0000 7f1da1e1c640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:47:11.671 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:11 smithi154 ceph-mon[65435]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.0 MiB/s wr, 493 op/s; 206/1629 objects degraded (12.646%) 2024-09-13T21:47:11.671 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:11 smithi154 ceph-mon[65435]: from='osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T21:47:11.672 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:11 smithi154 ceph-mon[65435]: osdmap e75: 8 total, 7 up, 8 in 2024-09-13T21:47:11.672 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:11 smithi154 ceph-mon[65435]: from='osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:47:11.672 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:11 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 206/1629 objects degraded (12.646%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:11.672 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:11 smithi154 ceph-mon[65435]: from='osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102]' entity='osd.0' 2024-09-13T21:47:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:11 smithi181 ceph-mon[51676]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.0 MiB/s wr, 493 op/s; 206/1629 objects degraded (12.646%) 2024-09-13T21:47:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:11 smithi181 ceph-mon[51676]: from='osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T21:47:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:11 smithi181 ceph-mon[51676]: osdmap e75: 8 total, 7 up, 8 in 2024-09-13T21:47:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:11 smithi181 ceph-mon[51676]: from='osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:47:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:11 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 206/1629 objects degraded (12.646%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:11 smithi181 ceph-mon[51676]: from='osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102]' entity='osd.0' 2024-09-13T21:47:11.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 168443 11.13 MB/sec execute 264 sec latency 38.432 ms 2024-09-13T21:47:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:12 smithi154 ceph-mon[65435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:47:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:12 smithi154 ceph-mon[65435]: osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102] boot 2024-09-13T21:47:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:12 smithi154 ceph-mon[65435]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T21:47:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:12 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:47:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:12 smithi181 ceph-mon[51676]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:47:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:12 smithi181 ceph-mon[51676]: osd.0 [v2:172.21.15.154:6802/3226237102,v1:172.21.15.154:6803/3226237102] boot 2024-09-13T21:47:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:12 smithi181 ceph-mon[51676]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T21:47:12.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:12 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T21:47:12.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 168496 11.10 MB/sec execute 265 sec latency 886.599 ms 2024-09-13T21:47:13.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 584 op/s; 212/1614 objects degraded (13.135%) 2024-09-13T21:47:13.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T21:47:13.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:13.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:13.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:13.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:13.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:13.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:13.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:13.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:47:13.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:47:13.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:47:13.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T21:47:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 584 op/s; 212/1614 objects degraded (13.135%) 2024-09-13T21:47:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T21:47:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:47:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:13.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:13.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:13.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:47:13.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:47:13.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:47:13.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T21:47:13.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 168496 11.06 MB/sec execute 266 sec latency 1886.626 ms 2024-09-13T21:47:14.781 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:14 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T21:47:14.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:14 smithi154 ceph-mon[65435]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T21:47:14.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:14 smithi154 ceph-mon[65435]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T21:47:14.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:14 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T21:47:14.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:14 smithi181 ceph-mon[51676]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T21:47:14.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:14 smithi181 ceph-mon[51676]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T21:47:14.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 168559 11.02 MB/sec execute 267 sec latency 2575.815 ms 2024-09-13T21:47:15.559 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:15 smithi154 ceph-mon[65435]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.9 MiB/s wr, 558 op/s; 212/1614 objects degraded (13.135%) 2024-09-13T21:47:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:15 smithi181 ceph-mon[51676]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.9 MiB/s wr, 558 op/s; 212/1614 objects degraded (13.135%) 2024-09-13T21:47:15.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 168891 11.02 MB/sec execute 268 sec latency 51.562 ms 2024-09-13T21:47:16.570 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:16 smithi154 ceph-mon[65435]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T21:47:16.570 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:16 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 212/1614 objects degraded (13.135%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:16 smithi181 ceph-mon[51676]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T21:47:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:16 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 212/1614 objects degraded (13.135%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:16.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 169221 10.99 MB/sec execute 269 sec latency 329.739 ms 2024-09-13T21:47:17.595 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:17 smithi154 ceph-mon[65435]: pgmap v73: 97 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 78 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.4 KiB/s rd, 2.1 MiB/s wr, 237 op/s; 154/1623 objects degraded (9.489%); 961 KiB/s, 17 objects/s recovering 2024-09-13T21:47:17.595 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:17 smithi154 ceph-mon[65435]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T21:47:17.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:17 smithi181 ceph-mon[51676]: pgmap v73: 97 pgs: 2 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 78 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.4 KiB/s rd, 2.1 MiB/s wr, 237 op/s; 154/1623 objects degraded (9.489%); 961 KiB/s, 17 objects/s recovering 2024-09-13T21:47:17.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:17 smithi181 ceph-mon[51676]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T21:47:17.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 169431 10.96 MB/sec execute 270 sec latency 989.136 ms 2024-09-13T21:47:18.618 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:18 smithi154 ceph-mon[65435]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T21:47:18.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:18 smithi181 ceph-mon[51676]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T21:47:18.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 169784 10.98 MB/sec execute 271 sec latency 749.493 ms 2024-09-13T21:47:19.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:19 smithi154 ceph-mon[65435]: pgmap v76: 97 pgs: 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.4 KiB/s rd, 2.8 MiB/s wr, 275 op/s; 186/1635 objects degraded (11.376%); 962 KiB/s, 34 objects/s recovering 2024-09-13T21:47:19.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:19 smithi154 ceph-mon[65435]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T21:47:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:19 smithi181 ceph-mon[51676]: pgmap v76: 97 pgs: 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.4 KiB/s rd, 2.8 MiB/s wr, 275 op/s; 186/1635 objects degraded (11.376%); 962 KiB/s, 34 objects/s recovering 2024-09-13T21:47:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:19 smithi181 ceph-mon[51676]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T21:47:19.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 169927 10.95 MB/sec execute 272 sec latency 981.884 ms 2024-09-13T21:47:20.575 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:20 smithi154 ceph-mon[65435]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T21:47:20.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:20 smithi181 ceph-mon[51676]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T21:47:20.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 170429 10.95 MB/sec execute 273 sec latency 47.714 ms 2024-09-13T21:47:21.582 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:21 smithi154 ceph-mon[65435]: pgmap v79: 97 pgs: 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 792 KiB/s wr, 41 op/s; 186/1635 objects degraded (11.376%); 22 B/s, 19 objects/s recovering 2024-09-13T21:47:21.582 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:21 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 186/1635 objects degraded (11.376%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:21.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:21 smithi181 ceph-mon[51676]: pgmap v79: 97 pgs: 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 792 KiB/s wr, 41 op/s; 186/1635 objects degraded (11.376%); 22 B/s, 19 objects/s recovering 2024-09-13T21:47:21.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:21 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 186/1635 objects degraded (11.376%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:21.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 170770 10.95 MB/sec execute 274 sec latency 34.497 ms 2024-09-13T21:47:22.855 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:22 smithi154 ceph-mon[65435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 186/1635 objects degraded (11.376%), 7 pgs degraded) 2024-09-13T21:47:22.855 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:22 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:47:22.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:22 smithi181 ceph-mon[51676]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 186/1635 objects degraded (11.376%), 7 pgs degraded) 2024-09-13T21:47:22.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:22 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:47:22.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 171093 10.95 MB/sec execute 275 sec latency 37.764 ms 2024-09-13T21:47:23.861 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:23 smithi154 ceph-mon[65435]: pgmap v80: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.6 MiB/s wr, 367 op/s; 0 B/s, 32 objects/s recovering 2024-09-13T21:47:23.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:23 smithi181 ceph-mon[51676]: pgmap v80: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.6 MiB/s wr, 367 op/s; 0 B/s, 32 objects/s recovering 2024-09-13T21:47:23.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 171409 10.93 MB/sec execute 276 sec latency 41.003 ms 2024-09-13T21:47:24.666 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:47:24.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 171715 10.93 MB/sec execute 277 sec latency 56.254 ms 2024-09-13T21:47:25.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:25 smithi154 ceph-mon[65435]: pgmap v81: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.1 MiB/s wr, 314 op/s; 0 B/s, 28 objects/s recovering 2024-09-13T21:47:25.644 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:25 smithi154 ceph-mon[65435]: from='client.44167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:25.644 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:25.644 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:47:25.777 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:47:25.777 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (4m) 18s ago 13m 27.1M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:47:25.777 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (67s) 18s ago 13m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:47:25.777 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (62s) 57s ago 12m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:47:25.777 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (4m) 18s ago 12m 53.5M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:47:25.778 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (7m) 18s ago 7m 37.2M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:47:25.778 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (7m) 57s ago 7m 16.9M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:47:25.778 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (3m) 18s ago 14m 497M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:47:25.778 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (2m) 57s ago 12m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:47:25.778 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (2m) 18s ago 14m 45.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:47:25.778 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (118s) 57s ago 12m 36.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:47:25.779 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (6m) 18s ago 6m 260M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:47:25.779 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (6m) 57s ago 6m 81.9M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:47:25.779 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (4m) 18s ago 12m 15.5M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:47:25.779 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (3m) 57s ago 12m 15.1M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:47:25.779 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (27s) 18s ago 11m 12.0M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:47:25.779 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (11m) 18s ago 11m 483M 1936M 17.2.0 e1d6a67b021e fe592dca8f7e 2024-09-13T21:47:25.780 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (10m) 18s ago 10m 405M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:47:25.780 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (10m) 18s ago 10m 512M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:47:25.780 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (9m) 57s ago 9m 489M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:47:25.780 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (9m) 57s ago 9m 310M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:47:25.780 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (9m) 57s ago 9m 491M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:47:25.780 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (8m) 57s ago 8m 374M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:47:25.780 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (2m) 18s ago 12m 64.4M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:47:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:25 smithi181 ceph-mon[51676]: pgmap v81: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.1 MiB/s wr, 314 op/s; 0 B/s, 28 objects/s recovering 2024-09-13T21:47:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:25 smithi181 ceph-mon[51676]: from='client.44167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:47:25.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 171978 10.91 MB/sec execute 278 sec latency 46.622 ms 2024-09-13T21:47:26.411 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:47:26.412 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:47:26.412 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:47:26.412 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:47:26.412 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:47:26.412 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:47:26.412 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:47:26.412 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:47:26.412 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T21:47:26.413 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T21:47:26.413 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:47:26.413 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:47:26.413 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:47:26.413 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:47:26.413 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:47:26.413 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-13T21:47:26.414 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T21:47:26.414 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:47:26.414 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:47:26.659 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:26 smithi154 ceph-mon[65435]: from='client.44171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:26.659 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:26 smithi154 ceph-mon[65435]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:26.659 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:26 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/3786534766' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:47:26.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:26 smithi181 ceph-mon[51676]: from='client.44171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:26.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:26 smithi181 ceph-mon[51676]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:26.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:26 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/3786534766' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:47:26.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 172220 10.89 MB/sec execute 279 sec latency 66.271 ms 2024-09-13T21:47:26.921 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:47:26.921 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:47:26.921 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:47:26.921 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "7/23 daemons upgraded", 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:47:26.922 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:47:27.559 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:47:27.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:27 smithi154 ceph-mon[65435]: pgmap v82: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.7 MiB/s wr, 410 op/s; 0 B/s, 24 objects/s recovering 2024-09-13T21:47:27.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:27 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/4014357963' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:47:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:27 smithi181 ceph-mon[51676]: pgmap v82: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.7 MiB/s wr, 410 op/s; 0 B/s, 24 objects/s recovering 2024-09-13T21:47:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:27 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/4014357963' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:47:27.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 172565 10.87 MB/sec execute 280 sec latency 51.914 ms 2024-09-13T21:47:28.673 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:28 smithi154 ceph-mon[65435]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:28.673 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:28 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T21:47:28.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:28 smithi181 ceph-mon[51676]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:28.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:28 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T21:47:28.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 172851 10.86 MB/sec execute 281 sec latency 50.653 ms 2024-09-13T21:47:29.703 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:29 smithi154 ceph-mon[65435]: pgmap v83: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 457 op/s; 0 B/s, 19 objects/s recovering 2024-09-13T21:47:29.703 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:29 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T21:47:29.704 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:29 smithi154 ceph-mon[65435]: Upgrade: osd.1 is safe to restart 2024-09-13T21:47:29.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:29 smithi181 ceph-mon[51676]: pgmap v83: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 457 op/s; 0 B/s, 19 objects/s recovering 2024-09-13T21:47:29.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:29 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T21:47:29.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:29 smithi181 ceph-mon[51676]: Upgrade: osd.1 is safe to restart 2024-09-13T21:47:29.902 INFO:teuthology.orchestra.run.smithi154.stdout: 5 173310 10.88 MB/sec execute 282 sec latency 46.025 ms 2024-09-13T21:47:30.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 173874 10.90 MB/sec execute 283 sec latency 64.149 ms 2024-09-13T21:47:31.730 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:31 smithi154 ceph-mon[65435]: pgmap v84: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 415 op/s; 0 B/s, 17 objects/s recovering 2024-09-13T21:47:31.730 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:31 smithi154 ceph-mon[65435]: Upgrade: Updating osd.1 2024-09-13T21:47:31.731 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:31 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:31.731 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:31 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:47:31.731 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:31 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:31.731 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:31 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:31.731 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:31 smithi154 ceph-mon[65435]: Deploying daemon osd.1 on smithi154 2024-09-13T21:47:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:31 smithi181 ceph-mon[51676]: pgmap v84: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 415 op/s; 0 B/s, 17 objects/s recovering 2024-09-13T21:47:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:31 smithi181 ceph-mon[51676]: Upgrade: Updating osd.1 2024-09-13T21:47:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:31 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:31 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T21:47:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:31 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:31 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:47:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:31 smithi181 ceph-mon[51676]: Deploying daemon osd.1 on smithi154 2024-09-13T21:47:31.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 174236 10.89 MB/sec execute 284 sec latency 43.935 ms 2024-09-13T21:47:32.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 174426 10.87 MB/sec execute 285 sec latency 75.887 ms 2024-09-13T21:47:33.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:33 smithi154 ceph-mon[65435]: pgmap v85: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 604 op/s; 0 B/s, 16 objects/s recovering 2024-09-13T21:47:33.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:33 smithi181 ceph-mon[51676]: pgmap v85: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 604 op/s; 0 B/s, 16 objects/s recovering 2024-09-13T21:47:33.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 174838 10.88 MB/sec execute 286 sec latency 54.650 ms 2024-09-13T21:47:34.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 175350 10.89 MB/sec execute 287 sec latency 38.543 ms 2024-09-13T21:47:35.014 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:34 smithi154 systemd[1]: Stopping Ceph osd.1 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:47:35.521 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:35 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[36987]: 2024-09-13T21:47:35.439+0000 7fe55a633700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:47:35.521 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:35 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[36987]: 2024-09-13T21:47:35.439+0000 7fe55a633700 -1 osd.1 83 *** Got signal Terminated *** 2024-09-13T21:47:35.521 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:35 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[36987]: 2024-09-13T21:47:35.439+0000 7fe55a633700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:47:35.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:35 smithi154 ceph-mon[65435]: pgmap v86: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 423 op/s 2024-09-13T21:47:35.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:35 smithi154 ceph-mon[65435]: osd.1 marked itself down and dead 2024-09-13T21:47:35.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:35 smithi181 ceph-mon[51676]: pgmap v86: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 423 op/s 2024-09-13T21:47:35.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:35 smithi181 ceph-mon[51676]: osd.1 marked itself down and dead 2024-09-13T21:47:35.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 175581 10.87 MB/sec execute 288 sec latency 398.873 ms 2024-09-13T21:47:36.410 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:36 smithi154 podman[73601]: 2024-09-13 21:47:36.152793337 +0000 UTC m=+0.927307104 container died fe592dca8f7e45c9c50ef38b6fd47085eb938af2887c6e1def79cf96c134453d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, version=8, ceph=True, RELEASE=HEAD, release=754, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-type=git) 2024-09-13T21:47:36.668 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:36 smithi154 podman[73601]: 2024-09-13 21:47:36.525876527 +0000 UTC m=+1.300390285 container cleanup fe592dca8f7e45c9c50ef38b6fd47085eb938af2887c6e1def79cf96c134453d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, ceph=True) 2024-09-13T21:47:36.669 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:36 smithi154 bash[73601]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1 2024-09-13T21:47:36.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:36 smithi154 ceph-mon[65435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:47:36.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:36 smithi154 ceph-mon[65435]: osdmap e84: 8 total, 7 up, 8 in 2024-09-13T21:47:36.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:36 smithi154 ceph-mon[65435]: osdmap e85: 8 total, 7 up, 8 in 2024-09-13T21:47:36.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:36 smithi181 ceph-mon[51676]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:47:36.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:36 smithi181 ceph-mon[51676]: osdmap e84: 8 total, 7 up, 8 in 2024-09-13T21:47:36.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:36 smithi181 ceph-mon[51676]: osdmap e85: 8 total, 7 up, 8 in 2024-09-13T21:47:36.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 175693 10.84 MB/sec execute 289 sec latency 1039.881 ms 2024-09-13T21:47:37.172 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:36 smithi154 podman[73617]: 2024-09-13 21:47:36.928612029 +0000 UTC m=+0.771522846 container remove fe592dca8f7e45c9c50ef38b6fd47085eb938af2887c6e1def79cf96c134453d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc.) 2024-09-13T21:47:37.423 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:37 smithi154 podman[73703]: 2024-09-13 21:47:37.251903805 +0000 UTC m=+0.024393337 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:47:37.424 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:37 smithi154 podman[73703]: 2024-09-13 21:47:37.388062863 +0000 UTC m=+0.160552388 container create 2dde9cd3335f7da41bed0451ee2db6d3dd3fa448170fca1dc42dc3ed261b1d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T21:47:37.677 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:37 smithi154 ceph-mon[65435]: pgmap v88: 97 pgs: 39 peering, 58 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 627 op/s 2024-09-13T21:47:37.677 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:37 smithi154 ceph-mon[65435]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-13T21:47:37.678 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:37 smithi154 podman[73703]: 2024-09-13 21:47:37.658140027 +0000 UTC m=+0.430629550 container init 2dde9cd3335f7da41bed0451ee2db6d3dd3fa448170fca1dc42dc3ed261b1d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T21:47:37.678 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:37 smithi154 podman[73703]: 2024-09-13 21:47:37.666877995 +0000 UTC m=+0.439367525 container start 2dde9cd3335f7da41bed0451ee2db6d3dd3fa448170fca1dc42dc3ed261b1d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-13T21:47:37.801 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:37 smithi181 ceph-mon[51676]: pgmap v88: 97 pgs: 39 peering, 58 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 627 op/s 2024-09-13T21:47:37.801 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:37 smithi181 ceph-mon[51676]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-13T21:47:37.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 176044 10.83 MB/sec execute 290 sec latency 80.617 ms 2024-09-13T21:47:37.928 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:37 smithi154 podman[73703]: 2024-09-13 21:47:37.796932433 +0000 UTC m=+0.569421979 container attach 2dde9cd3335f7da41bed0451ee2db6d3dd3fa448170fca1dc42dc3ed261b1d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T21:47:38.181 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:38 smithi154 podman[73703]: 2024-09-13 21:47:38.02790528 +0000 UTC m=+0.800394807 container died 2dde9cd3335f7da41bed0451ee2db6d3dd3fa448170fca1dc42dc3ed261b1d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:47:38.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 176554 10.83 MB/sec execute 291 sec latency 60.380 ms 2024-09-13T21:47:38.943 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:38 smithi154 podman[73726]: 2024-09-13 21:47:38.891816596 +0000 UTC m=+0.852479802 container remove 2dde9cd3335f7da41bed0451ee2db6d3dd3fa448170fca1dc42dc3ed261b1d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True) 2024-09-13T21:47:38.943 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:38 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T21:47:38.955 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:38 smithi154 systemd[1]: Stopped Ceph osd.1 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:47:38.955 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:38 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.1.service: Consumed 2min 57.257s CPU time. 2024-09-13T21:47:39.446 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:39 smithi154 systemd[1]: Starting Ceph osd.1 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:47:39.698 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:39 smithi154 podman[73819]: 2024-09-13 21:47:39.46372181 +0000 UTC m=+0.026960312 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:47:39.699 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:39 smithi154 ceph-mon[65435]: pgmap v90: 97 pgs: 39 peering, 58 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 629 op/s 2024-09-13T21:47:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:39 smithi181 ceph-mon[51676]: pgmap v90: 97 pgs: 39 peering, 58 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 629 op/s 2024-09-13T21:47:39.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 176971 10.85 MB/sec execute 292 sec latency 42.252 ms 2024-09-13T21:47:40.210 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:40 smithi154 podman[73819]: 2024-09-13 21:47:40.010320936 +0000 UTC m=+0.573559419 container create c9e7baaf7d105cc475bd69ef6aaaf8947ea63a1b5d9a179a51c199ae51856b37 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T21:47:40.714 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:40 smithi154 podman[73819]: 2024-09-13 21:47:40.478487889 +0000 UTC m=+1.041726372 container init c9e7baaf7d105cc475bd69ef6aaaf8947ea63a1b5d9a179a51c199ae51856b37 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:47:40.714 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:40 smithi154 podman[73819]: 2024-09-13 21:47:40.494589124 +0000 UTC m=+1.057827612 container start c9e7baaf7d105cc475bd69ef6aaaf8947ea63a1b5d9a179a51c199ae51856b37 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-13T21:47:40.714 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:40 smithi154 podman[73819]: 2024-09-13 21:47:40.641398072 +0000 UTC m=+1.204636569 container attach c9e7baaf7d105cc475bd69ef6aaaf8947ea63a1b5d9a179a51c199ae51856b37 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T21:47:40.714 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:40 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:47:40.714 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:40 smithi154 bash[73819]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:47:40.714 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:40 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:47:40.714 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:40 smithi154 bash[73819]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:47:40.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 177330 10.84 MB/sec execute 293 sec latency 64.639 ms 2024-09-13T21:47:41.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:41 smithi154 ceph-mon[65435]: pgmap v91: 97 pgs: 39 peering, 58 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.0 MiB/s wr, 288 op/s 2024-09-13T21:47:41.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:41.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:47:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:41 smithi181 ceph-mon[51676]: pgmap v91: 97 pgs: 39 peering, 58 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.0 MiB/s wr, 288 op/s 2024-09-13T21:47:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:47:41.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 177722 10.84 MB/sec execute 294 sec latency 53.659 ms 2024-09-13T21:47:41.955 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:41 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T21:47:41.955 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:41 smithi154 bash[73819]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T21:47:41.955 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:41 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T21:47:41.955 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:41 smithi154 bash[73819]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 bash[73819]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 bash[73819]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 bash[73819]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 bash[73819]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T21:47:42.501 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate[73832]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T21:47:42.502 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 bash[73819]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T21:47:42.502 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:42 smithi154 podman[73819]: 2024-09-13 21:47:42.337392298 +0000 UTC m=+2.900630783 container died c9e7baaf7d105cc475bd69ef6aaaf8947ea63a1b5d9a179a51c199ae51856b37 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T21:47:42.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:42 smithi154 ceph-mon[65435]: Health check failed: Degraded data redundancy: 195/1647 objects degraded (11.840%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:42.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:42 smithi154 ceph-mon[65435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-09-13T21:47:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:42 smithi181 ceph-mon[51676]: Health check failed: Degraded data redundancy: 195/1647 objects degraded (11.840%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:42 smithi181 ceph-mon[51676]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-09-13T21:47:42.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 178047 10.84 MB/sec execute 295 sec latency 43.213 ms 2024-09-13T21:47:43.508 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:43 smithi154 podman[73972]: 2024-09-13 21:47:43.33228318 +0000 UTC m=+0.987479532 container remove c9e7baaf7d105cc475bd69ef6aaaf8947ea63a1b5d9a179a51c199ae51856b37 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T21:47:43.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:43 smithi154 ceph-mon[65435]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.3 MiB/s wr, 634 op/s; 195/1647 objects degraded (11.840%) 2024-09-13T21:47:43.770 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:43 smithi154 podman[74063]: 2024-09-13 21:47:43.488540932 +0000 UTC m=+0.026436852 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:47:43.770 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:43 smithi154 podman[74063]: 2024-09-13 21:47:43.690096761 +0000 UTC m=+0.227992684 container create 91c024a3fc2e74b8b9b05e36faeece13a7feea0525b43383dfc939f304c2a7df (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T21:47:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:43 smithi181 ceph-mon[51676]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.3 MiB/s wr, 634 op/s; 195/1647 objects degraded (11.840%) 2024-09-13T21:47:43.903 INFO:teuthology.orchestra.run.smithi154.stdout: 5 178412 10.84 MB/sec execute 296 sec latency 36.831 ms 2024-09-13T21:47:44.271 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:44 smithi154 podman[74063]: 2024-09-13 21:47:44.24729085 +0000 UTC m=+0.785186769 container init 91c024a3fc2e74b8b9b05e36faeece13a7feea0525b43383dfc939f304c2a7df (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T21:47:44.271 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:44 smithi154 podman[74063]: 2024-09-13 21:47:44.261616204 +0000 UTC m=+0.799512115 container start 91c024a3fc2e74b8b9b05e36faeece13a7feea0525b43383dfc939f304c2a7df (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-13T21:47:44.527 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:44 smithi154 ceph-osd[74080]: -- 172.21.15.154:0/1657903777 <== mon.0 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56380a8d61e0 con 0x563809c92000 2024-09-13T21:47:44.778 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:44 smithi154 bash[74063]: 91c024a3fc2e74b8b9b05e36faeece13a7feea0525b43383dfc939f304c2a7df 2024-09-13T21:47:44.779 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:44 smithi154 systemd[1]: Started Ceph osd.1 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:47:44.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 178937 10.85 MB/sec execute 297 sec latency 39.971 ms 2024-09-13T21:47:45.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 179531 10.86 MB/sec execute 298 sec latency 36.279 ms 2024-09-13T21:47:45.964 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:45 smithi154 ceph-mon[65435]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 567 op/s; 195/1647 objects degraded (11.840%) 2024-09-13T21:47:45.964 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:45.964 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:45.964 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:45 smithi181 ceph-mon[51676]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 567 op/s; 195/1647 objects degraded (11.840%) 2024-09-13T21:47:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:47:46.473 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:46 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[74076]: 2024-09-13T21:47:46.427+0000 7fe2679bc640 -1 Falling back to public interface 2024-09-13T21:47:46.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 179874 10.84 MB/sec execute 299 sec latency 85.244 ms 2024-09-13T21:47:47.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 180517 10.88 MB/sec execute 300 sec latency 68.008 ms 2024-09-13T21:47:48.021 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:47 smithi154 ceph-mon[65435]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.9 MiB/s wr, 593 op/s; 210/1758 objects degraded (11.945%) 2024-09-13T21:47:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:47 smithi181 ceph-mon[51676]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.9 MiB/s wr, 593 op/s; 210/1758 objects degraded (11.945%) 2024-09-13T21:47:48.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 181124 10.89 MB/sec execute 301 sec latency 84.481 ms 2024-09-13T21:47:49.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 181446 10.88 MB/sec execute 302 sec latency 44.238 ms 2024-09-13T21:47:50.054 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:49 smithi154 ceph-mon[65435]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 496 op/s; 210/1758 objects degraded (11.945%) 2024-09-13T21:47:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:49 smithi181 ceph-mon[51676]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 496 op/s; 210/1758 objects degraded (11.945%) 2024-09-13T21:47:50.820 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:50 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 210/1758 objects degraded (11.945%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:50.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 181759 10.89 MB/sec execute 303 sec latency 93.181 ms 2024-09-13T21:47:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:50 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 210/1758 objects degraded (11.945%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:51.847 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:51 smithi154 ceph-mon[65435]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 494 op/s; 210/1758 objects degraded (11.945%) 2024-09-13T21:47:51.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 182080 10.88 MB/sec execute 304 sec latency 61.584 ms 2024-09-13T21:47:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:51 smithi181 ceph-mon[51676]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 494 op/s; 210/1758 objects degraded (11.945%) 2024-09-13T21:47:52.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 182381 10.87 MB/sec execute 305 sec latency 62.140 ms 2024-09-13T21:47:53.867 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:53 smithi154 ceph-mon[65435]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 765 op/s; 183/1650 objects degraded (11.091%) 2024-09-13T21:47:53.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 182825 10.87 MB/sec execute 306 sec latency 48.025 ms 2024-09-13T21:47:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:53 smithi181 ceph-mon[51676]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 765 op/s; 183/1650 objects degraded (11.091%) 2024-09-13T21:47:54.373 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:54 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[74076]: 2024-09-13T21:47:54.126+0000 7fe2679bc640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-13T21:47:54.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 183222 10.86 MB/sec execute 307 sec latency 42.160 ms 2024-09-13T21:47:55.139 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:55.139 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:55.139 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:55 smithi154 ceph-mon[65435]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 534 op/s; 183/1650 objects degraded (11.091%) 2024-09-13T21:47:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:55.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:55 smithi181 ceph-mon[51676]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 534 op/s; 183/1650 objects degraded (11.091%) 2024-09-13T21:47:55.406 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:55 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[74076]: 2024-09-13T21:47:55.395+0000 7fe2679bc640 -1 osd.1 83 log_to_monitors true 2024-09-13T21:47:55.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 183510 10.84 MB/sec execute 308 sec latency 57.994 ms 2024-09-13T21:47:56.411 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:47:56 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[74076]: 2024-09-13T21:47:56.246+0000 7fe25f767640 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:47:56.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:56 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:56.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:56 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:56.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:56 smithi154 ceph-mon[65435]: from='osd.1 [v2:172.21.15.154:6810/21416871,v1:172.21.15.154:6811/21416871]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:47:56.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:56 smithi154 ceph-mon[65435]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:47:56.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:56 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:56.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:56 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:47:56.412 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:56 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 183/1650 objects degraded (11.091%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:56 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:56 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:56 smithi181 ceph-mon[51676]: from='osd.1 [v2:172.21.15.154:6810/21416871,v1:172.21.15.154:6811/21416871]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:47:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:56 smithi181 ceph-mon[51676]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T21:47:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:56 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:47:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:56 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:47:56.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:56 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 183/1650 objects degraded (11.091%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T21:47:56.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 183836 10.85 MB/sec execute 309 sec latency 53.614 ms 2024-09-13T21:47:57.442 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:57 smithi154 ceph-mon[65435]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T21:47:57.442 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:57 smithi154 ceph-mon[65435]: from='osd.1 [v2:172.21.15.154:6810/21416871,v1:172.21.15.154:6811/21416871]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:47:57.442 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:57 smithi154 ceph-mon[65435]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T21:47:57.443 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:57 smithi154 ceph-mon[65435]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:47:57.443 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:57 smithi154 ceph-mon[65435]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 584 op/s; 180/1605 objects degraded (11.215%) 2024-09-13T21:47:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:57 smithi181 ceph-mon[51676]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T21:47:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:57 smithi181 ceph-mon[51676]: from='osd.1 [v2:172.21.15.154:6810/21416871,v1:172.21.15.154:6811/21416871]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:47:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:57 smithi181 ceph-mon[51676]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T21:47:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:57 smithi181 ceph-mon[51676]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:47:57.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:57 smithi181 ceph-mon[51676]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.0 MiB/s wr, 584 op/s; 180/1605 objects degraded (11.215%) 2024-09-13T21:47:57.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 184045 10.83 MB/sec execute 310 sec latency 594.402 ms 2024-09-13T21:47:58.064 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:47:58.201 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:58 smithi154 ceph-mon[65435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:47:58.202 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:58 smithi154 ceph-mon[65435]: osd.1 [v2:172.21.15.154:6810/21416871,v1:172.21.15.154:6811/21416871] boot 2024-09-13T21:47:58.202 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:58 smithi154 ceph-mon[65435]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T21:47:58.202 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:58 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:47:58.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:58 smithi181 ceph-mon[51676]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:47:58.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:58 smithi181 ceph-mon[51676]: osd.1 [v2:172.21.15.154:6810/21416871,v1:172.21.15.154:6811/21416871] boot 2024-09-13T21:47:58.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:58 smithi181 ceph-mon[51676]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T21:47:58.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:58 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (5m) 4s ago 13m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (100s) 4s ago 13m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (95s) 90s ago 13m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (4m) 4s ago 13m 53.0M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (7m) 4s ago 7m 37.4M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (7m) 90s ago 7m 16.9M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (4m) 4s ago 14m 501M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (3m) 90s ago 12m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (2m) 4s ago 14m 47.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:47:58.870 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (2m) 90s ago 12m 36.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (7m) 4s ago 7m 261M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (7m) 90s ago 7m 81.9M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (4m) 4s ago 13m 15.9M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (4m) 90s ago 12m 15.1M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (60s) 4s ago 12m 237M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (15s) 4s ago 11m 11.3M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (11m) 4s ago 11m 452M 1936M 17.2.0 e1d6a67b021e f0a8263c60e6 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (10m) 4s ago 10m 560M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (10m) 90s ago 10m 489M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (10m) 90s ago 10m 310M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (9m) 90s ago 9m 491M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (9m) 90s ago 9m 374M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:47:58.871 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (3m) 4s ago 12m 65.0M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:47:58.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 184098 10.80 MB/sec execute 311 sec latency 1594.432 ms 2024-09-13T21:47:59.347 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:47:59.348 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:47:59.349 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:47:59.349 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:47:59.349 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:47:59.349 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-13T21:47:59.349 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T21:47:59.349 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:47:59.349 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:47:59.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:59 smithi154 ceph-mon[65435]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:59.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:59 smithi154 ceph-mon[65435]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T21:47:59.456 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:59 smithi154 ceph-mon[65435]: pgmap v103: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 433 op/s; 180/1605 objects degraded (11.215%) 2024-09-13T21:47:59.457 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:47:59 smithi154 ceph-mon[65435]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:59 smithi181 ceph-mon[51676]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:59 smithi181 ceph-mon[51676]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T21:47:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:59 smithi181 ceph-mon[51676]: pgmap v103: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 433 op/s; 180/1605 objects degraded (11.215%) 2024-09-13T21:47:59.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:47:59 smithi181 ceph-mon[51676]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:47:59.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 184473 10.80 MB/sec execute 312 sec latency 1983.186 ms 2024-09-13T21:47:59.939 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "8/23 daemons upgraded", 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:47:59.940 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:48:00.219 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:00 smithi154 ceph-mon[65435]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:00.220 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:00 smithi154 ceph-mon[65435]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T21:48:00.220 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:00 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2521136859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:00.595 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_WARN Degraded data redundancy: 180/1605 objects degraded (11.215%), 24 pgs degraded 2024-09-13T21:48:00.595 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 180/1605 objects degraded (11.215%), 24 pgs degraded 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-13T21:48:00.596 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-13T21:48:00.597 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-13T21:48:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:00 smithi181 ceph-mon[51676]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:00 smithi181 ceph-mon[51676]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T21:48:00.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:00 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2521136859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:00.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 184887 10.79 MB/sec execute 313 sec latency 63.616 ms 2024-09-13T21:48:01.245 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: pgmap v106: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 180/1605 objects degraded (11.215%) 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/4128710375' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 180/1605 objects degraded (11.215%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:01.246 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:01.247 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:01.247 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:01.247 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: pgmap v106: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 180/1605 objects degraded (11.215%) 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/4128710375' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 180/1605 objects degraded (11.215%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:01.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:01.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T21:48:01.904 INFO:teuthology.orchestra.run.smithi154.stdout: 5 185134 10.79 MB/sec execute 314 sec latency 669.019 ms 2024-09-13T21:48:02.268 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:02 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T21:48:02.268 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:02 smithi154 ceph-mon[65435]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T21:48:02.269 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:02 smithi154 ceph-mon[65435]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T21:48:02.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:02 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T21:48:02.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:02 smithi181 ceph-mon[51676]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T21:48:02.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:02 smithi181 ceph-mon[51676]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T21:48:02.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 185381 10.76 MB/sec execute 315 sec latency 1019.009 ms 2024-09-13T21:48:03.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:03 smithi154 ceph-mon[65435]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T21:48:03.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:03 smithi154 ceph-mon[65435]: pgmap v109: 97 pgs: 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 84 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 170/1602 objects degraded (10.612%); 1024 KiB/s, 70 keys/s, 29 objects/s recovering 2024-09-13T21:48:03.300 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:03 smithi154 ceph-mon[65435]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T21:48:03.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:03 smithi181 ceph-mon[51676]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T21:48:03.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:03 smithi181 ceph-mon[51676]: pgmap v109: 97 pgs: 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 84 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 170/1602 objects degraded (10.612%); 1024 KiB/s, 70 keys/s, 29 objects/s recovering 2024-09-13T21:48:03.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:03 smithi181 ceph-mon[51676]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T21:48:03.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 185559 10.75 MB/sec execute 316 sec latency 957.925 ms 2024-09-13T21:48:04.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 185824 10.75 MB/sec execute 317 sec latency 1061.599 ms 2024-09-13T21:48:05.272 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:05 smithi154 ceph-mon[65435]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T21:48:05.272 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:05 smithi154 ceph-mon[65435]: pgmap v112: 97 pgs: 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 84 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 170/1602 objects degraded (10.612%); 1024 KiB/s, 70 keys/s, 29 objects/s recovering 2024-09-13T21:48:05.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:05 smithi181 ceph-mon[51676]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T21:48:05.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:05 smithi181 ceph-mon[51676]: pgmap v112: 97 pgs: 3 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 84 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 170/1602 objects degraded (10.612%); 1024 KiB/s, 70 keys/s, 29 objects/s recovering 2024-09-13T21:48:05.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 186123 10.73 MB/sec execute 318 sec latency 597.074 ms 2024-09-13T21:48:06.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:06 smithi154 ceph-mon[65435]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T21:48:06.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:06 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 170/1602 objects degraded (10.612%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:06.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:06 smithi181 ceph-mon[51676]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T21:48:06.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:06 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 170/1602 objects degraded (10.612%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:06.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 186430 10.74 MB/sec execute 319 sec latency 942.597 ms 2024-09-13T21:48:07.328 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:07 smithi154 ceph-mon[65435]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T21:48:07.329 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:07 smithi154 ceph-mon[65435]: pgmap v115: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 456 op/s; 1.7 MiB/s, 54 objects/s recovering 2024-09-13T21:48:07.550 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:07 smithi181 ceph-mon[51676]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T21:48:07.551 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:07 smithi181 ceph-mon[51676]: pgmap v115: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 456 op/s; 1.7 MiB/s, 54 objects/s recovering 2024-09-13T21:48:07.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 186804 10.74 MB/sec execute 320 sec latency 36.229 ms 2024-09-13T21:48:08.357 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:08 smithi154 ceph-mon[65435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 170/1602 objects degraded (10.612%), 7 pgs degraded) 2024-09-13T21:48:08.357 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:08 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:48:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:08 smithi181 ceph-mon[51676]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 170/1602 objects degraded (10.612%), 7 pgs degraded) 2024-09-13T21:48:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:08 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:48:08.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 187175 10.72 MB/sec execute 321 sec latency 45.587 ms 2024-09-13T21:48:09.368 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:09 smithi154 ceph-mon[65435]: pgmap v116: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 348 op/s; 1.3 MiB/s, 41 objects/s recovering 2024-09-13T21:48:09.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:09 smithi181 ceph-mon[51676]: pgmap v116: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 348 op/s; 1.3 MiB/s, 41 objects/s recovering 2024-09-13T21:48:09.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 187514 10.71 MB/sec execute 322 sec latency 32.113 ms 2024-09-13T21:48:10.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 187774 10.70 MB/sec execute 323 sec latency 35.293 ms 2024-09-13T21:48:11.706 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:11 smithi154 ceph-mon[65435]: pgmap v117: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.3 MiB/s wr, 292 op/s; 1.1 MiB/s, 34 objects/s recovering 2024-09-13T21:48:11.706 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:11.706 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:48:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:11 smithi181 ceph-mon[51676]: pgmap v117: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.3 MiB/s wr, 292 op/s; 1.1 MiB/s, 34 objects/s recovering 2024-09-13T21:48:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:11.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:48:11.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 188097 10.71 MB/sec execute 324 sec latency 40.630 ms 2024-09-13T21:48:12.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 188408 10.70 MB/sec execute 325 sec latency 72.345 ms 2024-09-13T21:48:13.522 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:13 smithi154 ceph-mon[65435]: pgmap v118: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 552 op/s; 892 KiB/s, 27 objects/s recovering 2024-09-13T21:48:13.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:13 smithi181 ceph-mon[51676]: pgmap v118: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 552 op/s; 892 KiB/s, 27 objects/s recovering 2024-09-13T21:48:13.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 188697 10.68 MB/sec execute 326 sec latency 59.021 ms 2024-09-13T21:48:14.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 188917 10.66 MB/sec execute 327 sec latency 38.479 ms 2024-09-13T21:48:15.572 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:15 smithi154 ceph-mon[65435]: pgmap v119: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 478 op/s; 773 KiB/s, 23 objects/s recovering 2024-09-13T21:48:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:15 smithi181 ceph-mon[51676]: pgmap v119: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 478 op/s; 773 KiB/s, 23 objects/s recovering 2024-09-13T21:48:15.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 189253 10.66 MB/sec execute 328 sec latency 45.728 ms 2024-09-13T21:48:16.629 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:16 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:16.629 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:16 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T21:48:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:16 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:16 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T21:48:16.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 189475 10.65 MB/sec execute 329 sec latency 210.225 ms 2024-09-13T21:48:17.654 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:17 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T21:48:17.655 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:17 smithi154 ceph-mon[65435]: Upgrade: osd.2 is safe to restart 2024-09-13T21:48:17.655 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:17 smithi154 ceph-mon[65435]: pgmap v120: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.7 MiB/s wr, 590 op/s; 0 B/s, 8 objects/s recovering 2024-09-13T21:48:17.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:17 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T21:48:17.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:17 smithi181 ceph-mon[51676]: Upgrade: osd.2 is safe to restart 2024-09-13T21:48:17.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:17 smithi181 ceph-mon[51676]: pgmap v120: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.7 MiB/s wr, 590 op/s; 0 B/s, 8 objects/s recovering 2024-09-13T21:48:17.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 189760 10.65 MB/sec execute 330 sec latency 76.237 ms 2024-09-13T21:48:18.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 190147 10.66 MB/sec execute 331 sec latency 89.451 ms 2024-09-13T21:48:19.449 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:19 smithi154 ceph-mon[65435]: Upgrade: Updating osd.2 2024-09-13T21:48:19.449 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:19.449 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:48:19.449 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:19.449 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:19 smithi154 ceph-mon[65435]: Deploying daemon osd.2 on smithi154 2024-09-13T21:48:19.449 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:19 smithi154 ceph-mon[65435]: pgmap v121: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 379 op/s 2024-09-13T21:48:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:19 smithi181 ceph-mon[51676]: Upgrade: Updating osd.2 2024-09-13T21:48:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T21:48:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:19 smithi181 ceph-mon[51676]: Deploying daemon osd.2 on smithi154 2024-09-13T21:48:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:19 smithi181 ceph-mon[51676]: pgmap v121: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 379 op/s 2024-09-13T21:48:19.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 190536 10.64 MB/sec execute 332 sec latency 75.943 ms 2024-09-13T21:48:20.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 190867 10.62 MB/sec execute 333 sec latency 66.084 ms 2024-09-13T21:48:21.554 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:21 smithi154 ceph-mon[65435]: pgmap v122: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 379 op/s 2024-09-13T21:48:21.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:21 smithi181 ceph-mon[51676]: pgmap v122: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 379 op/s 2024-09-13T21:48:21.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 191179 10.63 MB/sec execute 334 sec latency 51.758 ms 2024-09-13T21:48:22.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 191520 10.63 MB/sec execute 335 sec latency 45.415 ms 2024-09-13T21:48:23.088 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:23 smithi154 systemd[1]: Stopping Ceph osd.2 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:48:23.637 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:23 smithi154 ceph-mon[65435]: pgmap v123: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 588 op/s 2024-09-13T21:48:23.637 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:23 smithi154 ceph-mon[65435]: osd.2 marked itself down and dead 2024-09-13T21:48:23.638 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:23 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[40861]: 2024-09-13T21:48:23.417+0000 7f469795e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:48:23.638 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:23 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[40861]: 2024-09-13T21:48:23.417+0000 7f469795e700 -1 osd.2 96 *** Got signal Terminated *** 2024-09-13T21:48:23.638 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:23 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[40861]: 2024-09-13T21:48:23.417+0000 7f469795e700 -1 osd.2 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:48:23.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:23 smithi181 ceph-mon[51676]: pgmap v123: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 588 op/s 2024-09-13T21:48:23.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:23 smithi181 ceph-mon[51676]: osd.2 marked itself down and dead 2024-09-13T21:48:23.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 191838 10.62 MB/sec execute 336 sec latency 303.383 ms 2024-09-13T21:48:24.484 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:24 smithi154 podman[76194]: 2024-09-13 21:48:24.201229353 +0000 UTC m=+0.916871542 container stop f0a8263c60e632be537c9d811901efd3f9f9ba6c30ecf7a06041d7d5c57f166d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, release=754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, RELEASE=HEAD, ceph=True, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-type=git, io.k8s.display-name=CentOS Stream 8) 2024-09-13T21:48:24.484 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:24 smithi154 podman[76194]: 2024-09-13 21:48:24.226879113 +0000 UTC m=+0.942521301 container died f0a8263c60e632be537c9d811901efd3f9f9ba6c30ecf7a06041d7d5c57f166d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, vcs-type=git, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_BRANCH=HEAD, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True) 2024-09-13T21:48:24.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:24 smithi154 ceph-mon[65435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:48:24.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:24 smithi154 ceph-mon[65435]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T21:48:24.735 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:24 smithi154 podman[76194]: 2024-09-13 21:48:24.566169865 +0000 UTC m=+1.281812054 container cleanup f0a8263c60e632be537c9d811901efd3f9f9ba6c30ecf7a06041d7d5c57f166d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, vendor=Red Hat, Inc., release=754, io.openshift.expose-services=, RELEASE=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.component=centos-stream-container, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream) 2024-09-13T21:48:24.735 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:24 smithi154 bash[76194]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2 2024-09-13T21:48:24.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:24 smithi181 ceph-mon[51676]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:48:24.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:24 smithi181 ceph-mon[51676]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T21:48:24.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 192026 10.61 MB/sec execute 337 sec latency 980.832 ms 2024-09-13T21:48:24.985 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:24 smithi154 podman[76212]: 2024-09-13 21:48:24.792951529 +0000 UTC m=+0.585473480 container remove f0a8263c60e632be537c9d811901efd3f9f9ba6c30ecf7a06041d7d5c57f166d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.19.8, vcs-type=git, name=centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T21:48:25.488 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:25 smithi154 podman[76296]: 2024-09-13 21:48:25.255313073 +0000 UTC m=+0.026270388 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:48:25.488 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:25 smithi154 podman[76296]: 2024-09-13 21:48:25.413478485 +0000 UTC m=+0.184435777 container create df83475e33495ec78909df2ec18a3d573248c62090d52a9163628257e8bc9bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-13T21:48:25.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:25 smithi154 ceph-mon[65435]: pgmap v125: 97 pgs: 8 stale+active+clean, 89 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 446 op/s 2024-09-13T21:48:25.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:25 smithi154 ceph-mon[65435]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T21:48:25.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:25.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:48:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:25 smithi181 ceph-mon[51676]: pgmap v125: 97 pgs: 8 stale+active+clean, 89 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 446 op/s 2024-09-13T21:48:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:25 smithi181 ceph-mon[51676]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T21:48:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:48:25.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 192493 10.60 MB/sec execute 338 sec latency 51.836 ms 2024-09-13T21:48:25.933 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:25 smithi154 podman[76296]: 2024-09-13 21:48:25.885499411 +0000 UTC m=+0.656456709 container init df83475e33495ec78909df2ec18a3d573248c62090d52a9163628257e8bc9bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T21:48:25.933 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:25 smithi154 podman[76296]: 2024-09-13 21:48:25.889651407 +0000 UTC m=+0.660608700 container start df83475e33495ec78909df2ec18a3d573248c62090d52a9163628257e8bc9bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T21:48:26.243 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:26 smithi154 podman[76296]: 2024-09-13 21:48:26.069535776 +0000 UTC m=+0.840493072 container attach df83475e33495ec78909df2ec18a3d573248c62090d52a9163628257e8bc9bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:48:26.243 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:26 smithi154 podman[76296]: 2024-09-13 21:48:26.233461689 +0000 UTC m=+1.004418979 container died df83475e33495ec78909df2ec18a3d573248c62090d52a9163628257e8bc9bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T21:48:26.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:26 smithi154 ceph-mon[65435]: Health check failed: Degraded data redundancy: 127/1689 objects degraded (7.519%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:26.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:26 smithi181 ceph-mon[51676]: Health check failed: Degraded data redundancy: 127/1689 objects degraded (7.519%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:26.905 INFO:teuthology.orchestra.run.smithi154.stdout: 5 193110 10.62 MB/sec execute 339 sec latency 86.407 ms 2024-09-13T21:48:27.261 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:27 smithi154 podman[76318]: 2024-09-13 21:48:27.211683688 +0000 UTC m=+0.969589505 container remove df83475e33495ec78909df2ec18a3d573248c62090d52a9163628257e8bc9bc1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T21:48:27.518 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:27 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T21:48:27.518 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:27 smithi154 systemd[1]: Stopped Ceph osd.2 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:48:27.518 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:27 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 38.016s CPU time. 2024-09-13T21:48:27.769 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:27 smithi154 systemd[1]: Starting Ceph osd.2 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:48:27.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:27 smithi154 ceph-mon[65435]: pgmap v127: 97 pgs: 6 active+undersized, 3 stale+active+clean, 15 active+undersized+degraded, 73 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 6.1 MiB/s wr, 615 op/s; 127/1689 objects degraded (7.519%) 2024-09-13T21:48:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:27 smithi181 ceph-mon[51676]: pgmap v127: 97 pgs: 6 active+undersized, 3 stale+active+clean, 15 active+undersized+degraded, 73 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 6.1 MiB/s wr, 615 op/s; 127/1689 objects degraded (7.519%) 2024-09-13T21:48:27.906 INFO:teuthology.orchestra.run.smithi154.stdout: 5 193543 10.64 MB/sec execute 340 sec latency 41.572 ms 2024-09-13T21:48:28.019 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:27 smithi154 podman[76413]: 2024-09-13 21:48:27.771164337 +0000 UTC m=+0.024582826 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:48:28.019 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:27 smithi154 podman[76413]: 2024-09-13 21:48:27.973939403 +0000 UTC m=+0.227357889 container create cb8c167554d131536eb2553f8a98e63d21a8733e8fb1044c71a930d468cf1493 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True) 2024-09-13T21:48:28.524 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:28 smithi154 podman[76413]: 2024-09-13 21:48:28.385548263 +0000 UTC m=+0.638966751 container init cb8c167554d131536eb2553f8a98e63d21a8733e8fb1044c71a930d468cf1493 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:48:28.525 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:28 smithi154 podman[76413]: 2024-09-13 21:48:28.395937311 +0000 UTC m=+0.649355799 container start cb8c167554d131536eb2553f8a98e63d21a8733e8fb1044c71a930d468cf1493 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be) 2024-09-13T21:48:28.775 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:28 smithi154 podman[76413]: 2024-09-13 21:48:28.56128513 +0000 UTC m=+0.814703639 container attach cb8c167554d131536eb2553f8a98e63d21a8733e8fb1044c71a930d468cf1493 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T21:48:28.775 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:28 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:48:28.776 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:28 smithi154 bash[76413]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:48:28.776 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:28 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:48:28.776 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:28 smithi154 bash[76413]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:48:28.906 INFO:teuthology.orchestra.run.smithi154.stdout: 5 193931 10.64 MB/sec execute 341 sec latency 50.018 ms 2024-09-13T21:48:29.528 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T21:48:29.528 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 bash[76413]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T21:48:29.528 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T21:48:29.529 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 bash[76413]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T21:48:29.792 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:29 smithi154 ceph-mon[65435]: pgmap v128: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.5 MiB/s wr, 638 op/s; 151/1689 objects degraded (8.940%) 2024-09-13T21:48:29.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:29 smithi181 ceph-mon[51676]: pgmap v128: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.5 MiB/s wr, 638 op/s; 151/1689 objects degraded (8.940%) 2024-09-13T21:48:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: 5 194339 10.62 MB/sec execute 342 sec latency 43.478 ms 2024-09-13T21:48:30.056 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T21:48:30.056 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 bash[76413]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T21:48:30.056 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T21:48:30.056 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 bash[76413]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T21:48:30.056 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T21:48:30.056 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 bash[76413]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T21:48:30.056 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T21:48:30.057 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 bash[76413]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T21:48:30.057 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate[76427]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T21:48:30.057 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 bash[76413]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T21:48:30.057 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:29 smithi154 podman[76413]: 2024-09-13 21:48:29.902823465 +0000 UTC m=+2.156241954 container died cb8c167554d131536eb2553f8a98e63d21a8733e8fb1044c71a930d468cf1493 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-13T21:48:30.906 INFO:teuthology.orchestra.run.smithi154.stdout: 5 194670 10.62 MB/sec execute 343 sec latency 40.916 ms 2024-09-13T21:48:31.064 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:30 smithi154 podman[76562]: 2024-09-13 21:48:30.870236699 +0000 UTC m=+0.959382483 container remove cb8c167554d131536eb2553f8a98e63d21a8733e8fb1044c71a930d468cf1493 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T21:48:31.181 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:48:31.315 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:31 smithi154 podman[76639]: 2024-09-13 21:48:31.074014474 +0000 UTC m=+0.030275254 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:48:31.315 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:31 smithi154 podman[76639]: 2024-09-13 21:48:31.196545332 +0000 UTC m=+0.152806113 container create cc7d439fad81d5083252cbfba8f22f3dfc1217d5f7d828fc9453860eccdbf16e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T21:48:31.577 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:31 smithi154 podman[76639]: 2024-09-13 21:48:31.500143206 +0000 UTC m=+0.456403996 container init cc7d439fad81d5083252cbfba8f22f3dfc1217d5f7d828fc9453860eccdbf16e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T21:48:31.578 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:31 smithi154 podman[76639]: 2024-09-13 21:48:31.51411351 +0000 UTC m=+0.470374296 container start cc7d439fad81d5083252cbfba8f22f3dfc1217d5f7d828fc9453860eccdbf16e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T21:48:31.578 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:31 smithi154 ceph-osd[76681]: -- 172.21.15.154:0/3148567507 <== mon.0 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a2077221e0 con 0x55a2076f8000 2024-09-13T21:48:31.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:31 smithi154 ceph-mon[65435]: pgmap v129: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.4 MiB/s wr, 325 op/s; 151/1689 objects degraded (8.940%) 2024-09-13T21:48:31.839 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:31 smithi154 bash[76639]: cc7d439fad81d5083252cbfba8f22f3dfc1217d5f7d828fc9453860eccdbf16e 2024-09-13T21:48:31.839 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:31 smithi154 systemd[1]: Started Ceph osd.2 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:48:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:31 smithi181 ceph-mon[51676]: pgmap v129: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.4 MiB/s wr, 325 op/s; 151/1689 objects degraded (8.940%) 2024-09-13T21:48:31.914 INFO:teuthology.orchestra.run.smithi154.stdout: 5 195018 10.62 MB/sec execute 344 sec latency 60.801 ms 2024-09-13T21:48:32.357 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (5m) 38s ago 14m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (2m) 38s ago 14m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (2m) 2m ago 13m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (5m) 38s ago 13m 53.0M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (8m) 38s ago 8m 37.4M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (8m) 2m ago 8m 16.9M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (4m) 38s ago 15m 501M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (3m) 2m ago 13m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (3m) 38s ago 15m 47.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (3m) 2m ago 13m 36.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (7m) 38s ago 7m 261M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:48:32.358 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (7m) 2m ago 7m 81.9M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (5m) 38s ago 13m 15.9M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (4m) 2m ago 13m 15.1M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (94s) 38s ago 12m 237M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (48s) 38s ago 12m 11.3M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 starting - - - 1936M 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (11m) 38s ago 11m 560M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (10m) 2m ago 10m 489M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (10m) 2m ago 10m 310M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (10m) 2m ago 10m 491M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (9m) 2m ago 9m 374M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:48:32.359 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (3m) 38s ago 13m 65.0M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:48:32.859 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:32 smithi154 ceph-mon[65435]: from='client.34276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:32.860 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:32 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:32.860 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:32 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:32.860 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:32 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:32.860 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:32 smithi154 ceph-mon[65435]: from='client.44219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:32.914 INFO:teuthology.orchestra.run.smithi154.stdout: 5 195409 10.62 MB/sec execute 345 sec latency 65.050 ms 2024-09-13T21:48:33.096 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:48:33.096 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:48:33.096 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:48:33.097 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T21:48:33.098 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T21:48:33.098 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:48:33.098 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:48:33.125 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:33 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[76677]: 2024-09-13T21:48:33.107+0000 7f50de99f640 -1 Falling back to public interface 2024-09-13T21:48:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:32 smithi181 ceph-mon[51676]: from='client.34276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:32 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:32 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:32 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:32 smithi181 ceph-mon[51676]: from='client.44219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:48:33.612 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "8/23 daemons upgraded", 2024-09-13T21:48:33.613 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:48:33.613 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:48:33.613 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:48:33.876 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:33 smithi154 ceph-mon[65435]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:33.876 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:33 smithi154 ceph-mon[65435]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 603 op/s; 153/1659 objects degraded (9.222%) 2024-09-13T21:48:33.877 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:33 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 153/1659 objects degraded (9.222%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:33.877 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:33 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/389994571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:33.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 195797 10.62 MB/sec execute 346 sec latency 53.671 ms 2024-09-13T21:48:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:33 smithi181 ceph-mon[51676]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:33 smithi181 ceph-mon[51676]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 603 op/s; 153/1659 objects degraded (9.222%) 2024-09-13T21:48:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:33 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 153/1659 objects degraded (9.222%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:33 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/389994571' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 153/1659 objects degraded (9.222%), 19 pgs degraded 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout: osd.2 (root=default,host=smithi154) is down 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 153/1659 objects degraded (9.222%), 19 pgs degraded 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-13T21:48:34.206 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-13T21:48:34.207 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-13T21:48:34.887 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:34 smithi154 ceph-mon[65435]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:34.887 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:34 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2696508736' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:48:34.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 196163 10.61 MB/sec execute 347 sec latency 42.003 ms 2024-09-13T21:48:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:34 smithi181 ceph-mon[51676]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:48:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:34 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2696508736' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:48:35.901 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:35 smithi154 ceph-mon[65435]: pgmap v131: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 538 op/s; 153/1659 objects degraded (9.222%) 2024-09-13T21:48:35.914 INFO:teuthology.orchestra.run.smithi154.stdout: 5 196521 10.61 MB/sec execute 348 sec latency 30.703 ms 2024-09-13T21:48:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:35 smithi181 ceph-mon[51676]: pgmap v131: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 538 op/s; 153/1659 objects degraded (9.222%) 2024-09-13T21:48:36.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 196943 10.63 MB/sec execute 349 sec latency 29.948 ms 2024-09-13T21:48:37.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 197296 10.63 MB/sec execute 350 sec latency 68.447 ms 2024-09-13T21:48:37.944 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:37 smithi154 ceph-mon[65435]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 7.0 MiB/s wr, 684 op/s; 156/1680 objects degraded (9.286%) 2024-09-13T21:48:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:37 smithi181 ceph-mon[51676]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 7.0 MiB/s wr, 684 op/s; 156/1680 objects degraded (9.286%) 2024-09-13T21:48:38.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 197657 10.61 MB/sec execute 351 sec latency 52.992 ms 2024-09-13T21:48:38.948 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:38 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 159/1704 objects degraded (9.331%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:38 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 159/1704 objects degraded (9.331%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:39.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 197905 10.59 MB/sec execute 352 sec latency 47.751 ms 2024-09-13T21:48:39.959 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:39 smithi154 ceph-mon[65435]: pgmap v133: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.2 MiB/s wr, 493 op/s; 159/1704 objects degraded (9.331%) 2024-09-13T21:48:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:39 smithi181 ceph-mon[51676]: pgmap v133: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.2 MiB/s wr, 493 op/s; 159/1704 objects degraded (9.331%) 2024-09-13T21:48:40.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 198176 10.58 MB/sec execute 353 sec latency 84.340 ms 2024-09-13T21:48:40.989 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:40 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[76677]: 2024-09-13T21:48:40.803+0000 7f50de99f640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-13T21:48:41.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:41 smithi154 ceph-mon[65435]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 478 op/s; 159/1704 objects degraded (9.331%) 2024-09-13T21:48:41.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:41.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:48:41.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:41.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:41 smithi181 ceph-mon[51676]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 478 op/s; 159/1704 objects degraded (9.331%) 2024-09-13T21:48:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:48:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:41.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:41.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 198459 10.58 MB/sec execute 354 sec latency 48.756 ms 2024-09-13T21:48:42.006 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:41 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[76677]: 2024-09-13T21:48:41.805+0000 7f50de99f640 -1 osd.2 96 log_to_monitors true 2024-09-13T21:48:42.531 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:42 smithi154 ceph-mon[65435]: from='osd.2 [v2:172.21.15.154:6818/2770920231,v1:172.21.15.154:6819/2770920231]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:48:42.531 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:42 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:42.531 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:42 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:42.793 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:48:42 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[76677]: 2024-09-13T21:48:42.582+0000 7f50d674a640 -1 osd.2 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:48:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:42 smithi181 ceph-mon[51676]: from='osd.2 [v2:172.21.15.154:6818/2770920231,v1:172.21.15.154:6819/2770920231]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T21:48:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:42 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:42 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:42.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 198861 10.59 MB/sec execute 355 sec latency 60.436 ms 2024-09-13T21:48:43.548 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:43 smithi154 ceph-mon[65435]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 681 op/s; 149/1590 objects degraded (9.371%) 2024-09-13T21:48:43.548 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:43 smithi154 ceph-mon[65435]: from='osd.2 [v2:172.21.15.154:6818/2770920231,v1:172.21.15.154:6819/2770920231]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T21:48:43.548 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:43 smithi154 ceph-mon[65435]: osdmap e99: 8 total, 7 up, 8 in 2024-09-13T21:48:43.548 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:43 smithi154 ceph-mon[65435]: from='osd.2 [v2:172.21.15.154:6818/2770920231,v1:172.21.15.154:6819/2770920231]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:48:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:43 smithi181 ceph-mon[51676]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 681 op/s; 149/1590 objects degraded (9.371%) 2024-09-13T21:48:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:43 smithi181 ceph-mon[51676]: from='osd.2 [v2:172.21.15.154:6818/2770920231,v1:172.21.15.154:6819/2770920231]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T21:48:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:43 smithi181 ceph-mon[51676]: osdmap e99: 8 total, 7 up, 8 in 2024-09-13T21:48:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:43 smithi181 ceph-mon[51676]: from='osd.2 [v2:172.21.15.154:6818/2770920231,v1:172.21.15.154:6819/2770920231]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:48:43.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 199476 10.59 MB/sec execute 356 sec latency 372.419 ms 2024-09-13T21:48:44.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:44 smithi154 ceph-mon[65435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:48:44.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:44 smithi154 ceph-mon[65435]: osd.2 [v2:172.21.15.154:6818/2770920231,v1:172.21.15.154:6819/2770920231] boot 2024-09-13T21:48:44.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:44 smithi154 ceph-mon[65435]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T21:48:44.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:44 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:48:44.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:44 smithi154 ceph-mon[65435]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T21:48:44.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:44 smithi181 ceph-mon[51676]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:48:44.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:44 smithi181 ceph-mon[51676]: osd.2 [v2:172.21.15.154:6818/2770920231,v1:172.21.15.154:6819/2770920231] boot 2024-09-13T21:48:44.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:44 smithi181 ceph-mon[51676]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T21:48:44.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:44 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T21:48:44.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:44 smithi181 ceph-mon[51676]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T21:48:44.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 199535 10.56 MB/sec execute 357 sec latency 1372.452 ms 2024-09-13T21:48:45.661 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:45 smithi154 ceph-mon[65435]: pgmap v138: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 329 op/s; 149/1590 objects degraded (9.371%) 2024-09-13T21:48:45.661 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:45 smithi154 ceph-mon[65435]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T21:48:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:45 smithi181 ceph-mon[51676]: pgmap v138: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 329 op/s; 149/1590 objects degraded (9.371%) 2024-09-13T21:48:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:45 smithi181 ceph-mon[51676]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T21:48:45.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 199614 10.54 MB/sec execute 358 sec latency 2116.431 ms 2024-09-13T21:48:46.698 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:46 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 149/1590 objects degraded (9.371%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:46.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:46 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 149/1590 objects degraded (9.371%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:46.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 199910 10.53 MB/sec execute 359 sec latency 166.305 ms 2024-09-13T21:48:47.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:47 smithi154 ceph-mon[65435]: pgmap v141: 97 pgs: 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized, 5 active+undersized+degraded, 80 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 MiB/s wr, 521 op/s; 143/1464 objects degraded (9.768%); 1.0 MiB/s, 23 objects/s recovering 2024-09-13T21:48:47.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:47 smithi154 ceph-mon[65435]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T21:48:47.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:47 smithi181 ceph-mon[51676]: pgmap v141: 97 pgs: 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized, 5 active+undersized+degraded, 80 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 MiB/s wr, 521 op/s; 143/1464 objects degraded (9.768%); 1.0 MiB/s, 23 objects/s recovering 2024-09-13T21:48:47.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:47 smithi181 ceph-mon[51676]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T21:48:47.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 200092 10.53 MB/sec execute 360 sec latency 839.642 ms 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:48.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:48.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:48.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T21:48:48.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:48 smithi154 ceph-mon[65435]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T21:48:48.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T21:48:48.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:48.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:48.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:48:48.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:48:48.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:48.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:48.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:48:48.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:48.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:48.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:48:48.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T21:48:48.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:48 smithi181 ceph-mon[51676]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T21:48:48.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 200303 10.52 MB/sec execute 361 sec latency 896.908 ms 2024-09-13T21:48:49.594 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:49 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T21:48:49.594 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:49 smithi154 ceph-mon[65435]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T21:48:49.594 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:49 smithi154 ceph-mon[65435]: pgmap v144: 97 pgs: 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 526 op/s; 114/1464 objects degraded (7.787%); 1.0 MiB/s, 24 objects/s recovering 2024-09-13T21:48:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:49 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T21:48:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:49 smithi181 ceph-mon[51676]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T21:48:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:49 smithi181 ceph-mon[51676]: pgmap v144: 97 pgs: 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 526 op/s; 114/1464 objects degraded (7.787%); 1.0 MiB/s, 24 objects/s recovering 2024-09-13T21:48:49.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 200896 10.53 MB/sec execute 362 sec latency 40.043 ms 2024-09-13T21:48:50.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 201436 10.55 MB/sec execute 363 sec latency 23.011 ms 2024-09-13T21:48:51.719 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:51 smithi154 ceph-mon[65435]: pgmap v146: 97 pgs: 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.7 MiB/s wr, 425 op/s; 114/1464 objects degraded (7.787%); 860 KiB/s, 19 objects/s recovering 2024-09-13T21:48:51.719 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:51 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 114/1464 objects degraded (7.787%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:51.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:51 smithi181 ceph-mon[51676]: pgmap v146: 97 pgs: 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.7 MiB/s wr, 425 op/s; 114/1464 objects degraded (7.787%); 860 KiB/s, 19 objects/s recovering 2024-09-13T21:48:51.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:51 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 114/1464 objects degraded (7.787%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T21:48:51.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 201862 10.55 MB/sec execute 364 sec latency 31.376 ms 2024-09-13T21:48:52.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:52 smithi154 ceph-mon[65435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 114/1464 objects degraded (7.787%), 5 pgs degraded) 2024-09-13T21:48:52.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:52 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:48:52.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:52 smithi181 ceph-mon[51676]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 114/1464 objects degraded (7.787%), 5 pgs degraded) 2024-09-13T21:48:52.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:52 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:48:52.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 202332 10.57 MB/sec execute 365 sec latency 43.844 ms 2024-09-13T21:48:53.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:53 smithi154 ceph-mon[65435]: pgmap v147: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.2 MiB/s wr, 447 op/s; 25 KiB/s, 19 objects/s recovering 2024-09-13T21:48:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:53 smithi181 ceph-mon[51676]: pgmap v147: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.2 MiB/s wr, 447 op/s; 25 KiB/s, 19 objects/s recovering 2024-09-13T21:48:53.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 202902 10.56 MB/sec execute 366 sec latency 31.752 ms 2024-09-13T21:48:54.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 203314 10.55 MB/sec execute 367 sec latency 43.504 ms 2024-09-13T21:48:55.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:55 smithi154 ceph-mon[65435]: pgmap v148: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.0 MiB/s wr, 342 op/s; 19 KiB/s, 14 objects/s recovering 2024-09-13T21:48:55.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:55.773 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:48:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:55 smithi181 ceph-mon[51676]: pgmap v148: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.0 MiB/s wr, 342 op/s; 19 KiB/s, 14 objects/s recovering 2024-09-13T21:48:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:48:55.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:48:55.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 203835 10.57 MB/sec execute 368 sec latency 27.659 ms 2024-09-13T21:48:56.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 204335 10.58 MB/sec execute 369 sec latency 36.539 ms 2024-09-13T21:48:57.832 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:57 smithi154 ceph-mon[65435]: pgmap v149: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 6.2 MiB/s wr, 537 op/s; 3.1 KiB/s, 12 objects/s recovering 2024-09-13T21:48:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:57 smithi181 ceph-mon[51676]: pgmap v149: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 6.2 MiB/s wr, 537 op/s; 3.1 KiB/s, 12 objects/s recovering 2024-09-13T21:48:57.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 204693 10.58 MB/sec execute 370 sec latency 75.997 ms 2024-09-13T21:48:58.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 205136 10.59 MB/sec execute 371 sec latency 39.891 ms 2024-09-13T21:48:59.867 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:48:59 smithi154 ceph-mon[65435]: pgmap v150: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.6 MiB/s wr, 660 op/s; 2.8 KiB/s, 11 objects/s recovering 2024-09-13T21:48:59.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:48:59 smithi181 ceph-mon[51676]: pgmap v150: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.6 MiB/s wr, 660 op/s; 2.8 KiB/s, 11 objects/s recovering 2024-09-13T21:48:59.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 205488 10.59 MB/sec execute 372 sec latency 40.739 ms 2024-09-13T21:49:00.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 205702 10.58 MB/sec execute 373 sec latency 62.820 ms 2024-09-13T21:49:01.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:01 smithi154 ceph-mon[65435]: pgmap v151: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 554 op/s; 2.3 KiB/s, 9 objects/s recovering 2024-09-13T21:49:01.705 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:01 smithi181 ceph-mon[51676]: pgmap v151: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 554 op/s; 2.3 KiB/s, 9 objects/s recovering 2024-09-13T21:49:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:01.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 205909 10.57 MB/sec execute 374 sec latency 58.598 ms 2024-09-13T21:49:02.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 206125 10.55 MB/sec execute 375 sec latency 95.078 ms 2024-09-13T21:49:03.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:03 smithi154 ceph-mon[65435]: pgmap v152: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.8 MiB/s wr, 734 op/s; 2.3 KiB/s, 9 objects/s recovering 2024-09-13T21:49:03.785 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T21:49:03.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:03 smithi181 ceph-mon[51676]: pgmap v152: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.8 MiB/s wr, 734 op/s; 2.3 KiB/s, 9 objects/s recovering 2024-09-13T21:49:03.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T21:49:03.915 INFO:teuthology.orchestra.run.smithi154.stdout: 5 206413 10.52 MB/sec execute 376 sec latency 89.408 ms 2024-09-13T21:49:04.802 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:04 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T21:49:04.802 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:04 smithi154 ceph-mon[65435]: Upgrade: osd.3 is safe to restart 2024-09-13T21:49:04.846 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:49:04.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:04 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T21:49:04.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:04 smithi181 ceph-mon[51676]: Upgrade: osd.3 is safe to restart 2024-09-13T21:49:04.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 206652 10.51 MB/sec execute 377 sec latency 86.339 ms 2024-09-13T21:49:05.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:05 smithi154 ceph-mon[65435]: pgmap v153: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-09-13T21:49:05.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:05 smithi154 ceph-mon[65435]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:05.858 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:49:05.858 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (6m) 24s ago 14m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:49:05.858 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (2m) 24s ago 14m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:49:05.858 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (2m) 2m ago 14m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:49:05.858 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (5m) 24s ago 14m 53.0M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (8m) 24s ago 8m 37.3M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (8m) 2m ago 8m 16.9M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (5m) 24s ago 15m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (4m) 2m ago 14m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (4m) 24s ago 16m 50.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (3m) 2m ago 14m 36.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (8m) 24s ago 8m 263M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (8m) 2m ago 8m 81.9M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (5m) 24s ago 14m 15.4M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (5m) 2m ago 13m 15.1M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:49:05.859 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (2m) 24s ago 13m 289M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:49:05.860 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (82s) 24s ago 12m 213M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:49:05.860 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (34s) 24s ago 12m 12.1M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:49:05.860 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (11m) 24s ago 11m 355M 1936M 17.2.0 e1d6a67b021e 1bcf16e6f4d3 2024-09-13T21:49:05.860 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (11m) 2m ago 11m 489M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:49:05.860 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (11m) 2m ago 11m 310M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:49:05.860 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (10m) 2m ago 10m 491M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:49:05.860 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (10m) 2m ago 10m 374M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:49:05.860 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (4m) 24s ago 13m 65.4M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:49:05.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:05 smithi181 ceph-mon[51676]: pgmap v153: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-09-13T21:49:05.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:05 smithi181 ceph-mon[51676]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:05.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 206875 10.50 MB/sec execute 378 sec latency 70.800 ms 2024-09-13T21:49:06.486 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:49:06.486 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:49:06.486 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:49:06.486 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 7 2024-09-13T21:49:06.487 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:49:06.488 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:49:06.850 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:06 smithi154 ceph-mon[65435]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:06.850 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:06 smithi154 ceph-mon[65435]: Upgrade: Updating osd.3 2024-09-13T21:49:06.850 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:06.851 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:49:06.851 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:06.851 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:06 smithi154 ceph-mon[65435]: Deploying daemon osd.3 on smithi154 2024-09-13T21:49:06.851 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:06 smithi154 ceph-mon[65435]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:06.851 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:06 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2761966667' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:06 smithi181 ceph-mon[51676]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:06 smithi181 ceph-mon[51676]: Upgrade: Updating osd.3 2024-09-13T21:49:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T21:49:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:06 smithi181 ceph-mon[51676]: Deploying daemon osd.3 on smithi154 2024-09-13T21:49:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:06 smithi181 ceph-mon[51676]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:06.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:06 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2761966667' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:06.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 207163 10.50 MB/sec execute 379 sec latency 71.994 ms 2024-09-13T21:49:07.089 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:49:07.089 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "9/23 daemons upgraded", 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:49:07.090 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:49:07.723 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:49:07.891 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:07 smithi154 ceph-mon[65435]: pgmap v154: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.3 MiB/s wr, 653 op/s 2024-09-13T21:49:07.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:07 smithi181 ceph-mon[51676]: pgmap v154: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.3 MiB/s wr, 653 op/s 2024-09-13T21:49:07.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 207478 10.50 MB/sec execute 380 sec latency 56.151 ms 2024-09-13T21:49:08.655 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:08 smithi154 ceph-mon[65435]: from='client.44251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:08.656 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:08 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/348232095' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:49:08.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:08 smithi181 ceph-mon[51676]: from='client.44251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:08.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:08 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/348232095' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:49:08.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 207896 10.50 MB/sec execute 381 sec latency 46.786 ms 2024-09-13T21:49:09.694 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:09 smithi154 ceph-mon[65435]: pgmap v155: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.4 MiB/s wr, 492 op/s 2024-09-13T21:49:09.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:09 smithi181 ceph-mon[51676]: pgmap v155: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.4 MiB/s wr, 492 op/s 2024-09-13T21:49:09.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 208170 10.49 MB/sec execute 382 sec latency 75.750 ms 2024-09-13T21:49:10.196 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:10 smithi154 systemd[1]: Stopping Ceph osd.3 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:49:10.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:49:10.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:10 smithi154 ceph-mon[65435]: osd.3 marked itself down and dead 2024-09-13T21:49:10.839 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:10 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[44590]: 2024-09-13T21:49:10.567+0000 7f719b5cf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:49:10.839 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:10 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[44590]: 2024-09-13T21:49:10.567+0000 7f719b5cf700 -1 osd.3 105 *** Got signal Terminated *** 2024-09-13T21:49:10.839 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:10 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[44590]: 2024-09-13T21:49:10.567+0000 7f719b5cf700 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:49:10.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:49:10.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:10 smithi181 ceph-mon[51676]: osd.3 marked itself down and dead 2024-09-13T21:49:10.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 208363 10.48 MB/sec execute 383 sec latency 294.424 ms 2024-09-13T21:49:11.553 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:11 smithi154 podman[78931]: 2024-09-13 21:49:11.299010551 +0000 UTC m=+0.899161276 container died 1bcf16e6f4d3867622ae28eb226ab5e110465d81bdd9ce41e2da2d7c8e169258 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, version=8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, release=754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vendor=Red Hat, Inc., name=centos-stream) 2024-09-13T21:49:11.801 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:11 smithi154 ceph-mon[65435]: pgmap v156: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.5 MiB/s wr, 341 op/s 2024-09-13T21:49:11.801 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:11 smithi154 ceph-mon[65435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:49:11.801 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:11 smithi154 ceph-mon[65435]: osdmap e106: 8 total, 7 up, 8 in 2024-09-13T21:49:11.802 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:11 smithi154 podman[78931]: 2024-09-13 21:49:11.665438783 +0000 UTC m=+1.265589501 container cleanup 1bcf16e6f4d3867622ae28eb226ab5e110465d81bdd9ce41e2da2d7c8e169258 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vendor=Red Hat, Inc., architecture=x86_64, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, name=centos-stream, ceph=True, version=8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T21:49:11.802 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:11 smithi154 bash[78931]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3 2024-09-13T21:49:11.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 208503 10.47 MB/sec execute 384 sec latency 1052.141 ms 2024-09-13T21:49:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:11 smithi181 ceph-mon[51676]: pgmap v156: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.5 MiB/s wr, 341 op/s 2024-09-13T21:49:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:11 smithi181 ceph-mon[51676]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:49:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:11 smithi181 ceph-mon[51676]: osdmap e106: 8 total, 7 up, 8 in 2024-09-13T21:49:12.308 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:12 smithi154 podman[78947]: 2024-09-13 21:49:12.059663679 +0000 UTC m=+0.762155258 container remove 1bcf16e6f4d3867622ae28eb226ab5e110465d81bdd9ce41e2da2d7c8e169258 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_CLEAN=True, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vendor=Red Hat, Inc., version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream) 2024-09-13T21:49:12.567 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:12 smithi154 podman[79034]: 2024-09-13 21:49:12.409665647 +0000 UTC m=+0.019145550 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:49:12.567 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:12 smithi154 podman[79034]: 2024-09-13 21:49:12.544114846 +0000 UTC m=+0.153594741 container create 924e7bc1174d2a7b927e74f00b6d1cf6eff02a26dfb51bf8508ff6ce9bf66740 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:49:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:12 smithi154 ceph-mon[65435]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T21:49:12.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 208844 10.47 MB/sec execute 385 sec latency 94.467 ms 2024-09-13T21:49:13.097 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:12 smithi154 podman[79034]: 2024-09-13 21:49:12.996029799 +0000 UTC m=+0.605509705 container init 924e7bc1174d2a7b927e74f00b6d1cf6eff02a26dfb51bf8508ff6ce9bf66740 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T21:49:13.098 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:13 smithi154 podman[79034]: 2024-09-13 21:49:13.011934811 +0000 UTC m=+0.621414710 container start 924e7bc1174d2a7b927e74f00b6d1cf6eff02a26dfb51bf8508ff6ce9bf66740 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:49:13.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:12 smithi181 ceph-mon[51676]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T21:49:13.602 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:13 smithi154 podman[79034]: 2024-09-13 21:49:13.444854617 +0000 UTC m=+1.054334519 container attach 924e7bc1174d2a7b927e74f00b6d1cf6eff02a26dfb51bf8508ff6ce9bf66740 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True) 2024-09-13T21:49:13.603 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:13 smithi154 podman[79034]: 2024-09-13 21:49:13.445168966 +0000 UTC m=+1.054648876 container died 924e7bc1174d2a7b927e74f00b6d1cf6eff02a26dfb51bf8508ff6ce9bf66740 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-13T21:49:13.859 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:13 smithi154 ceph-mon[65435]: pgmap v159: 97 pgs: 27 peering, 4 stale+active+clean, 66 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 MiB/s wr, 467 op/s 2024-09-13T21:49:13.859 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:13 smithi154 ceph-mon[65435]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T21:49:13.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 209087 10.46 MB/sec execute 386 sec latency 51.271 ms 2024-09-13T21:49:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:13 smithi181 ceph-mon[51676]: pgmap v159: 97 pgs: 27 peering, 4 stale+active+clean, 66 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 MiB/s wr, 467 op/s 2024-09-13T21:49:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:13 smithi181 ceph-mon[51676]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T21:49:14.640 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:14 smithi154 podman[79056]: 2024-09-13 21:49:14.373661602 +0000 UTC m=+1.027308186 container remove 924e7bc1174d2a7b927e74f00b6d1cf6eff02a26dfb51bf8508ff6ce9bf66740 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T21:49:14.641 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:14 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T21:49:14.641 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:14 smithi154 systemd[1]: Stopped Ceph osd.3 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:49:14.641 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:14 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.3.service: Consumed 3min 49.556s CPU time. 2024-09-13T21:49:14.902 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:14 smithi154 ceph-mon[65435]: Health check failed: Degraded data redundancy: 13/1668 objects degraded (0.779%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:14.903 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:14 smithi154 systemd[1]: Starting Ceph osd.3 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:49:14.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 209250 10.44 MB/sec execute 387 sec latency 87.915 ms 2024-09-13T21:49:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:14 smithi181 ceph-mon[51676]: Health check failed: Degraded data redundancy: 13/1668 objects degraded (0.779%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:15.154 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:14 smithi154 podman[79150]: 2024-09-13 21:49:14.884670965 +0000 UTC m=+0.017978323 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:49:15.154 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:15 smithi154 podman[79150]: 2024-09-13 21:49:15.046884896 +0000 UTC m=+0.180192249 container create c1163a5b7fb48111f017971554a3a9331ab91e81afe472c29fa630dfe506e2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T21:49:15.407 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:15 smithi154 podman[79150]: 2024-09-13 21:49:15.388230993 +0000 UTC m=+0.521538351 container init c1163a5b7fb48111f017971554a3a9331ab91e81afe472c29fa630dfe506e2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T21:49:15.407 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:15 smithi154 podman[79150]: 2024-09-13 21:49:15.394416895 +0000 UTC m=+0.527724249 container start c1163a5b7fb48111f017971554a3a9331ab91e81afe472c29fa630dfe506e2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-13T21:49:15.673 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:15 smithi154 podman[79150]: 2024-09-13 21:49:15.547245978 +0000 UTC m=+0.680553332 container attach c1163a5b7fb48111f017971554a3a9331ab91e81afe472c29fa630dfe506e2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-13T21:49:15.674 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:15 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:49:15.674 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:15 smithi154 bash[79150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:49:15.674 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:15 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:49:15.674 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:15 smithi154 bash[79150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:49:15.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 209446 10.43 MB/sec execute 388 sec latency 60.459 ms 2024-09-13T21:49:15.936 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:15 smithi154 ceph-mon[65435]: pgmap v160: 97 pgs: 2 active+undersized, 27 peering, 3 stale+active+clean, 3 active+undersized+degraded, 62 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.0 MiB/s wr, 257 op/s; 13/1668 objects degraded (0.779%) 2024-09-13T21:49:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:15 smithi181 ceph-mon[51676]: pgmap v160: 97 pgs: 2 active+undersized, 27 peering, 3 stale+active+clean, 3 active+undersized+degraded, 62 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.0 MiB/s wr, 257 op/s; 13/1668 objects degraded (0.779%) 2024-09-13T21:49:16.187 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T21:49:16.188 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 bash[79150]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T21:49:16.188 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T21:49:16.188 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 bash[79150]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T21:49:16.698 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T21:49:16.698 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 bash[79150]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T21:49:16.698 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T21:49:16.698 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 bash[79150]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T21:49:16.698 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T21:49:16.698 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 bash[79150]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T21:49:16.698 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T21:49:16.698 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 bash[79150]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T21:49:16.699 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate[79165]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T21:49:16.699 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 bash[79150]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T21:49:16.699 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:16 smithi154 podman[79150]: 2024-09-13 21:49:16.557724084 +0000 UTC m=+1.691031438 container died c1163a5b7fb48111f017971554a3a9331ab91e81afe472c29fa630dfe506e2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T21:49:16.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 209671 10.41 MB/sec execute 389 sec latency 74.643 ms 2024-09-13T21:49:16.961 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:16 smithi154 ceph-mon[65435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-09-13T21:49:17.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:16 smithi181 ceph-mon[51676]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-09-13T21:49:17.469 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:17 smithi154 podman[79310]: 2024-09-13 21:49:17.390500797 +0000 UTC m=+0.826674237 container remove c1163a5b7fb48111f017971554a3a9331ab91e81afe472c29fa630dfe506e2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-activate, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, io.buildah.version=1.37.2) 2024-09-13T21:49:17.747 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:17 smithi154 podman[79367]: 2024-09-13 21:49:17.542189501 +0000 UTC m=+0.022714661 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:49:17.747 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:17 smithi154 podman[79367]: 2024-09-13 21:49:17.672341568 +0000 UTC m=+0.152866715 container create 287dcaef58cf28cfb35efcf116552d724635e73545be119b609fe99033535254 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:49:17.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 210025 10.40 MB/sec execute 390 sec latency 50.068 ms 2024-09-13T21:49:18.005 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:17 smithi154 ceph-mon[65435]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s wr, 468 op/s; 192/1563 objects degraded (12.284%) 2024-09-13T21:49:18.005 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:17 smithi154 podman[79367]: 2024-09-13 21:49:17.966909444 +0000 UTC m=+0.447434595 container init 287dcaef58cf28cfb35efcf116552d724635e73545be119b609fe99033535254 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be) 2024-09-13T21:49:18.006 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:17 smithi154 podman[79367]: 2024-09-13 21:49:17.979284976 +0000 UTC m=+0.459810135 container start 287dcaef58cf28cfb35efcf116552d724635e73545be119b609fe99033535254 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:49:18.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:17 smithi181 ceph-mon[51676]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s wr, 468 op/s; 192/1563 objects degraded (12.284%) 2024-09-13T21:49:18.256 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:18 smithi154 ceph-osd[79386]: -- 172.21.15.154:0/4085212407 <== mon.0 v2:172.21.15.154:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556e89bc21e0 con 0x556e89b98000 2024-09-13T21:49:18.256 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:18 smithi154 bash[79367]: 287dcaef58cf28cfb35efcf116552d724635e73545be119b609fe99033535254 2024-09-13T21:49:18.257 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:18 smithi154 systemd[1]: Started Ceph osd.3 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:49:18.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 210245 10.38 MB/sec execute 391 sec latency 48.507 ms 2024-09-13T21:49:19.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:19.318 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:19.318 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:19.318 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:19 smithi154 ceph-mon[65435]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.1 MiB/s wr, 470 op/s; 192/1563 objects degraded (12.284%) 2024-09-13T21:49:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:19.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:19 smithi181 ceph-mon[51676]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 5.1 MiB/s wr, 470 op/s; 192/1563 objects degraded (12.284%) 2024-09-13T21:49:19.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 210480 10.37 MB/sec execute 392 sec latency 66.394 ms 2024-09-13T21:49:20.103 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:20 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:49:20.081+0000 7f8af2500640 -1 Falling back to public interface 2024-09-13T21:49:20.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 210895 10.38 MB/sec execute 393 sec latency 40.777 ms 2024-09-13T21:49:21.620 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:21 smithi154 ceph-mon[65435]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.7 MiB/s wr, 364 op/s; 192/1563 objects degraded (12.284%) 2024-09-13T21:49:21.621 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:21 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 192/1563 objects degraded (12.284%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:21.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:21 smithi181 ceph-mon[51676]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.7 MiB/s wr, 364 op/s; 192/1563 objects degraded (12.284%) 2024-09-13T21:49:21.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:21 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 192/1563 objects degraded (12.284%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:21.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 211312 10.39 MB/sec execute 394 sec latency 57.677 ms 2024-09-13T21:49:22.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 211687 10.39 MB/sec execute 395 sec latency 41.419 ms 2024-09-13T21:49:23.647 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:23 smithi154 ceph-mon[65435]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.5 MiB/s wr, 376 op/s; 195/1554 objects degraded (12.548%) 2024-09-13T21:49:23.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:23 smithi181 ceph-mon[51676]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.5 MiB/s wr, 376 op/s; 195/1554 objects degraded (12.548%) 2024-09-13T21:49:23.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 211993 10.39 MB/sec execute 396 sec latency 43.324 ms 2024-09-13T21:49:24.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 212343 10.38 MB/sec execute 397 sec latency 88.158 ms 2024-09-13T21:49:25.664 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:25 smithi154 ceph-mon[65435]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.2 MiB/s wr, 341 op/s; 195/1551 objects degraded (12.573%) 2024-09-13T21:49:25.665 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:25.665 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:49:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:25 smithi181 ceph-mon[51676]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.2 MiB/s wr, 341 op/s; 195/1551 objects degraded (12.573%) 2024-09-13T21:49:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:49:25.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 212628 10.38 MB/sec execute 398 sec latency 51.357 ms 2024-09-13T21:49:26.794 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:26 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 195/1551 objects degraded (12.573%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:26.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:26 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 195/1551 objects degraded (12.573%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:26.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 212912 10.38 MB/sec execute 399 sec latency 64.024 ms 2024-09-13T21:49:27.568 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:27 smithi154 ceph-mon[65435]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.5 MiB/s wr, 539 op/s; 216/1692 objects degraded (12.766%) 2024-09-13T21:49:27.568 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:27.568 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:27 smithi181 ceph-mon[51676]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.5 MiB/s wr, 539 op/s; 216/1692 objects degraded (12.766%) 2024-09-13T21:49:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:27.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:27.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 213199 10.37 MB/sec execute 400 sec latency 54.339 ms 2024-09-13T21:49:28.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 213710 10.36 MB/sec execute 401 sec latency 39.541 ms 2024-09-13T21:49:29.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:28 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:29.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:28 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:29.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:28 smithi154 ceph-mon[65435]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 386 op/s; 216/1692 objects degraded (12.766%) 2024-09-13T21:49:29.091 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:29 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:49:29.021+0000 7f8af2500640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-13T21:49:29.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:28 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:29.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:28 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:29.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:28 smithi181 ceph-mon[51676]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 386 op/s; 216/1692 objects degraded (12.766%) 2024-09-13T21:49:29.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 214234 10.37 MB/sec execute 402 sec latency 38.878 ms 2024-09-13T21:49:30.642 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:30 smithi154 ceph-mon[65435]: from='osd.3 [v2:172.21.15.154:6826/2011541625,v1:172.21.15.154:6827/2011541625]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:49:30.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:30 smithi154 ceph-mon[65435]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:49:30.643 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:30 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:49:30.360+0000 7f8af2500640 -1 osd.3 105 log_to_monitors true 2024-09-13T21:49:30.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:30 smithi181 ceph-mon[51676]: from='osd.3 [v2:172.21.15.154:6826/2011541625,v1:172.21.15.154:6827/2011541625]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:49:30.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:30 smithi181 ceph-mon[51676]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T21:49:30.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 214542 10.37 MB/sec execute 403 sec latency 213.506 ms 2024-09-13T21:49:31.145 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:49:30 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:49:30.986+0000 7f8ae62a3640 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:49:31.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:31 smithi154 ceph-mon[65435]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T21:49:31.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:31 smithi154 ceph-mon[65435]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T21:49:31.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:31 smithi154 ceph-mon[65435]: from='osd.3 [v2:172.21.15.154:6826/2011541625,v1:172.21.15.154:6827/2011541625]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:49:31.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:31 smithi154 ceph-mon[65435]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:49:31.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:31 smithi154 ceph-mon[65435]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 461 op/s; 216/1692 objects degraded (12.766%) 2024-09-13T21:49:31.651 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:31 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 216/1692 objects degraded (12.766%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:31.651 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:31 smithi154 ceph-mon[65435]: from='osd.3 ' entity='osd.3' 2024-09-13T21:49:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:31 smithi181 ceph-mon[51676]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T21:49:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:31 smithi181 ceph-mon[51676]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T21:49:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:31 smithi181 ceph-mon[51676]: from='osd.3 [v2:172.21.15.154:6826/2011541625,v1:172.21.15.154:6827/2011541625]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:49:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:31 smithi181 ceph-mon[51676]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-09-13T21:49:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:31 smithi181 ceph-mon[51676]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 461 op/s; 216/1692 objects degraded (12.766%) 2024-09-13T21:49:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:31 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 216/1692 objects degraded (12.766%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:31 smithi181 ceph-mon[51676]: from='osd.3 ' entity='osd.3' 2024-09-13T21:49:31.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 215086 10.38 MB/sec execute 404 sec latency 40.310 ms 2024-09-13T21:49:32.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:32 smithi154 ceph-mon[65435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:49:32.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:32 smithi154 ceph-mon[65435]: osd.3 [v2:172.21.15.154:6826/2011541625,v1:172.21.15.154:6827/2011541625] boot 2024-09-13T21:49:32.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:32 smithi154 ceph-mon[65435]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T21:49:32.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:32 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:32.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:32 smithi181 ceph-mon[51676]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:49:32.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:32 smithi181 ceph-mon[51676]: osd.3 [v2:172.21.15.154:6826/2011541625,v1:172.21.15.154:6827/2011541625] boot 2024-09-13T21:49:32.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:32 smithi181 ceph-mon[51676]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T21:49:32.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:32 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T21:49:32.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 215224 10.36 MB/sec execute 405 sec latency 938.315 ms 2024-09-13T21:49:33.572 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:33 smithi154 ceph-mon[65435]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 6.7 MiB/s wr, 668 op/s; 211/1644 objects degraded (12.835%) 2024-09-13T21:49:33.572 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:33 smithi154 ceph-mon[65435]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T21:49:33.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:33.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:33.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:33.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:33.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:33.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:33 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:33.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:33 smithi181 ceph-mon[51676]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 6.7 MiB/s wr, 668 op/s; 211/1644 objects degraded (12.835%) 2024-09-13T21:49:33.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:33 smithi181 ceph-mon[51676]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T21:49:33.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:33.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:33.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:33.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:49:33.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:33.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:33 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:33.917 INFO:teuthology.orchestra.run.smithi154.stdout: 5 215265 10.34 MB/sec execute 406 sec latency 1938.342 ms 2024-09-13T21:49:34.617 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:34 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:34.617 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:34 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:34.617 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:34 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:34.617 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:34 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:34.617 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:34 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T21:49:34.617 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:34 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T21:49:34.617 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:34 smithi154 ceph-mon[65435]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T21:49:34.617 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:34 smithi154 ceph-mon[65435]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T21:49:34.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:34 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:49:34.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:34 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:34.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:34 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:34.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:34 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:34.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:34 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T21:49:34.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:34 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T21:49:34.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:34 smithi181 ceph-mon[51676]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T21:49:34.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:34 smithi181 ceph-mon[51676]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T21:49:34.917 INFO:teuthology.orchestra.run.smithi154.stdout: 5 215513 10.32 MB/sec execute 407 sec latency 2073.993 ms 2024-09-13T21:49:35.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:35 smithi154 ceph-mon[65435]: pgmap v174: 97 pgs: 4 peering, 12 active+undersized, 25 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 724 op/s; 199/1644 objects degraded (12.105%) 2024-09-13T21:49:35.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:35 smithi154 ceph-mon[65435]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T21:49:35.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:35 smithi181 ceph-mon[51676]: pgmap v174: 97 pgs: 4 peering, 12 active+undersized, 25 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 724 op/s; 199/1644 objects degraded (12.105%) 2024-09-13T21:49:35.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:35 smithi181 ceph-mon[51676]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T21:49:35.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 215600 10.30 MB/sec execute 408 sec latency 952.141 ms 2024-09-13T21:49:36.497 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:36 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 199/1644 objects degraded (12.105%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:36.497 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:36 smithi154 ceph-mon[65435]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T21:49:36.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:36 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 199/1644 objects degraded (12.105%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:36.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:36 smithi181 ceph-mon[51676]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T21:49:36.916 INFO:teuthology.orchestra.run.smithi154.stdout: 5 216237 10.34 MB/sec execute 409 sec latency 1027.600 ms 2024-09-13T21:49:37.501 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:37 smithi154 ceph-mon[65435]: pgmap v177: 97 pgs: 14 peering, 83 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 420 op/s; 786 KiB/s, 65 objects/s recovering 2024-09-13T21:49:37.800 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:37 smithi181 ceph-mon[51676]: pgmap v177: 97 pgs: 14 peering, 83 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 420 op/s; 786 KiB/s, 65 objects/s recovering 2024-09-13T21:49:37.917 INFO:teuthology.orchestra.run.smithi154.stdout: 5 216823 10.34 MB/sec execute 410 sec latency 33.279 ms 2024-09-13T21:49:38.223 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:49:38.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:38 smithi154 ceph-mon[65435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 199/1644 objects degraded (12.105%), 25 pgs degraded) 2024-09-13T21:49:38.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:38 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:49:38.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:38 smithi181 ceph-mon[51676]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 199/1644 objects degraded (12.105%), 25 pgs degraded) 2024-09-13T21:49:38.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:38 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:49:38.917 INFO:teuthology.orchestra.run.smithi154.stdout: 5 217055 10.33 MB/sec execute 411 sec latency 54.077 ms 2024-09-13T21:49:39.146 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:49:39.146 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (6m) 12s ago 15m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:49:39.146 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (3m) 12s ago 15m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:49:39.146 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (3m) 3m ago 14m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:49:39.146 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (6m) 12s ago 15m 52.7M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:49:39.146 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (9m) 12s ago 9m 37.3M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (9m) 3m ago 9m 16.9M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (5m) 12s ago 16m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (4m) 3m ago 14m 440M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (4m) 12s ago 16m 50.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (4m) 3m ago 14m 36.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (9m) 12s ago 9m 279M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (8m) 3m ago 8m 81.9M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (6m) 12s ago 15m 15.4M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (5m) 3m ago 14m 15.1M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (2m) 12s ago 13m 333M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (115s) 12s ago 13m 257M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (68s) 12s ago 12m 201M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:49:39.147 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (21s) 12s ago 12m 14.2M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:49:39.148 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (12m) 3m ago 12m 489M 2703M 17.2.0 e1d6a67b021e 1b0f428d8716 2024-09-13T21:49:39.148 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (11m) 3m ago 11m 310M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:49:39.148 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (11m) 3m ago 11m 491M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:49:39.148 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (10m) 3m ago 10m 374M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:49:39.148 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (5m) 12s ago 14m 66.1M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:49:39.613 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:39 smithi154 ceph-mon[65435]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:39.613 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:39 smithi154 ceph-mon[65435]: pgmap v178: 97 pgs: 10 peering, 87 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.3 MiB/s wr, 305 op/s; 588 KiB/s, 47 objects/s recovering 2024-09-13T21:49:39.613 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:39 smithi154 ceph-mon[65435]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:49:39.688 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:49:39.689 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:49:39.689 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:49:39.689 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:49:39.689 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T21:49:39.689 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T21:49:39.689 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:49:39.689 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:49:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:39 smithi181 ceph-mon[51676]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:39 smithi181 ceph-mon[51676]: pgmap v178: 97 pgs: 10 peering, 87 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.3 MiB/s wr, 305 op/s; 588 KiB/s, 47 objects/s recovering 2024-09-13T21:49:39.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:39 smithi181 ceph-mon[51676]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:39.917 INFO:teuthology.orchestra.run.smithi154.stdout: 5 217308 10.32 MB/sec execute 412 sec latency 89.220 ms 2024-09-13T21:49:40.160 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:49:40.160 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:49:40.160 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:49:40.160 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:49:40.160 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:49:40.160 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:49:40.160 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:49:40.160 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:49:40.161 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:49:40.161 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "10/23 daemons upgraded", 2024-09-13T21:49:40.161 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:49:40.161 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:49:40.161 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:49:40.632 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:40 smithi154 ceph-mon[65435]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:40.632 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:40 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/687312564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:40.803 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:49:40.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:40 smithi181 ceph-mon[51676]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:40.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:40 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/687312564' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:49:40.917 INFO:teuthology.orchestra.run.smithi154.stdout: 5 217639 10.32 MB/sec execute 413 sec latency 54.132 ms 2024-09-13T21:49:41.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:41 smithi154 ceph-mon[65435]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:41.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:41.670 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:49:41.670 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:41 smithi154 ceph-mon[65435]: pgmap v179: 97 pgs: 2 peering, 95 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.1 MiB/s wr, 280 op/s; 1.4 MiB/s, 48 objects/s recovering 2024-09-13T21:49:41.670 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:41 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/3658405566' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:49:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:41 smithi181 ceph-mon[51676]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:49:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:49:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:41 smithi181 ceph-mon[51676]: pgmap v179: 97 pgs: 2 peering, 95 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.1 MiB/s wr, 280 op/s; 1.4 MiB/s, 48 objects/s recovering 2024-09-13T21:49:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:41 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/3658405566' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:49:41.917 INFO:teuthology.orchestra.run.smithi154.stdout: 5 217921 10.31 MB/sec execute 414 sec latency 61.030 ms 2024-09-13T21:49:42.927 INFO:teuthology.orchestra.run.smithi154.stdout: 5 218188 10.31 MB/sec execute 415 sec latency 63.282 ms 2024-09-13T21:49:43.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:43 smithi154 ceph-mon[65435]: pgmap v180: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 5.7 MiB/s wr, 553 op/s; 1.2 MiB/s, 39 objects/s recovering 2024-09-13T21:49:43.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:43 smithi181 ceph-mon[51676]: pgmap v180: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 5.7 MiB/s wr, 553 op/s; 1.2 MiB/s, 39 objects/s recovering 2024-09-13T21:49:43.927 INFO:teuthology.orchestra.run.smithi154.stdout: 5 218415 10.30 MB/sec execute 416 sec latency 69.973 ms 2024-09-13T21:49:44.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 218707 10.29 MB/sec execute 417 sec latency 59.291 ms 2024-09-13T21:49:45.809 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:45 smithi154 ceph-mon[65435]: pgmap v181: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 466 op/s; 991 KiB/s, 32 objects/s recovering 2024-09-13T21:49:45.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:45 smithi181 ceph-mon[51676]: pgmap v181: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 466 op/s; 991 KiB/s, 32 objects/s recovering 2024-09-13T21:49:45.927 INFO:teuthology.orchestra.run.smithi154.stdout: 5 218971 10.28 MB/sec execute 418 sec latency 70.089 ms 2024-09-13T21:49:46.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 219266 10.28 MB/sec execute 419 sec latency 38.475 ms 2024-09-13T21:49:47.604 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:47 smithi154 ceph-mon[65435]: pgmap v182: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.6 MiB/s wr, 568 op/s; 662 KiB/s, 13 objects/s recovering 2024-09-13T21:49:47.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:47 smithi181 ceph-mon[51676]: pgmap v182: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.6 MiB/s wr, 568 op/s; 662 KiB/s, 13 objects/s recovering 2024-09-13T21:49:47.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 219521 10.27 MB/sec execute 420 sec latency 77.044 ms 2024-09-13T21:49:48.606 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:48 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T21:49:48.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:48 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T21:49:48.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 219865 10.28 MB/sec execute 421 sec latency 53.122 ms 2024-09-13T21:49:49.621 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:49 smithi154 ceph-mon[65435]: pgmap v183: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.9 MiB/s wr, 421 op/s; 526 KiB/s, 4 objects/s recovering 2024-09-13T21:49:49.621 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:49 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T21:49:49.621 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:49 smithi154 ceph-mon[65435]: Upgrade: osd.4 is safe to restart 2024-09-13T21:49:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:49 smithi181 ceph-mon[51676]: pgmap v183: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.9 MiB/s wr, 421 op/s; 526 KiB/s, 4 objects/s recovering 2024-09-13T21:49:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:49 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T21:49:49.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:49 smithi181 ceph-mon[51676]: Upgrade: osd.4 is safe to restart 2024-09-13T21:49:49.927 INFO:teuthology.orchestra.run.smithi154.stdout: 5 220210 10.27 MB/sec execute 422 sec latency 62.264 ms 2024-09-13T21:49:50.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 220430 10.25 MB/sec execute 423 sec latency 55.418 ms 2024-09-13T21:49:51.425 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:51 smithi154 ceph-mon[65435]: Upgrade: Updating osd.4 2024-09-13T21:49:51.425 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:51.425 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:49:51.425 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:51.425 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:51 smithi154 ceph-mon[65435]: Deploying daemon osd.4 on smithi181 2024-09-13T21:49:51.425 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:51 smithi154 ceph-mon[65435]: pgmap v184: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 429 op/s; 516 KiB/s, 4 objects/s recovering 2024-09-13T21:49:51.426 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:51 smithi181 ceph-mon[51676]: Upgrade: Updating osd.4 2024-09-13T21:49:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T21:49:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:49:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:51 smithi181 ceph-mon[51676]: Deploying daemon osd.4 on smithi181 2024-09-13T21:49:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:51 smithi181 ceph-mon[51676]: pgmap v184: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 429 op/s; 516 KiB/s, 4 objects/s recovering 2024-09-13T21:49:51.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:51.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 220599 10.24 MB/sec execute 424 sec latency 83.111 ms 2024-09-13T21:49:52.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 220794 10.23 MB/sec execute 425 sec latency 86.009 ms 2024-09-13T21:49:53.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:53 smithi181 ceph-mon[51676]: pgmap v185: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.3 MiB/s wr, 506 op/s 2024-09-13T21:49:53.723 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:53 smithi154 ceph-mon[65435]: pgmap v185: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.3 MiB/s wr, 506 op/s 2024-09-13T21:49:53.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 221022 10.22 MB/sec execute 426 sec latency 127.022 ms 2024-09-13T21:49:54.088 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:53 smithi181 systemd[1]: Stopping Ceph osd.4 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:49:54.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:54 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[30051]: 2024-09-13T21:49:54.164+0000 7f7ecc57c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:49:54.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:54 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[30051]: 2024-09-13T21:49:54.164+0000 7f7ecc57c700 -1 osd.4 113 *** Got signal Terminated *** 2024-09-13T21:49:54.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:54 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[30051]: 2024-09-13T21:49:54.164+0000 7f7ecc57c700 -1 osd.4 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:49:54.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:54 smithi154 ceph-mon[65435]: osd.4 marked itself down and dead 2024-09-13T21:49:54.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:54 smithi181 ceph-mon[51676]: osd.4 marked itself down and dead 2024-09-13T21:49:54.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 221212 10.21 MB/sec execute 427 sec latency 370.948 ms 2024-09-13T21:49:55.457 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:55 smithi181 podman[56080]: 2024-09-13 21:49:55.181863152 +0000 UTC m=+1.152202235 container died 1b0f428d8716caab4d01d65d03e6bfe8d2b1ee9b10683e47d595b5905cd6c57e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, distribution-scope=public, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, version=8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, release=754) 2024-09-13T21:49:55.817 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:55 smithi154 ceph-mon[65435]: pgmap v186: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.4 MiB/s wr, 339 op/s 2024-09-13T21:49:55.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:55 smithi154 ceph-mon[65435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:49:55.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:55 smithi154 ceph-mon[65435]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T21:49:55.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:55.818 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:49:55.842 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:55 smithi181 ceph-mon[51676]: pgmap v186: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.4 MiB/s wr, 339 op/s 2024-09-13T21:49:55.842 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:55 smithi181 ceph-mon[51676]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:49:55.842 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:55 smithi181 ceph-mon[51676]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T21:49:55.843 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:49:55.843 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:49:55.843 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:55 smithi181 podman[56080]: 2024-09-13 21:49:55.547226369 +0000 UTC m=+1.517565443 container cleanup 1b0f428d8716caab4d01d65d03e6bfe8d2b1ee9b10683e47d595b5905cd6c57e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, ceph=True, vendor=Red Hat, Inc., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T21:49:55.843 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:55 smithi181 bash[56080]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4 2024-09-13T21:49:55.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 221347 10.19 MB/sec execute 428 sec latency 1066.474 ms 2024-09-13T21:49:56.150 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:55 smithi181 podman[56095]: 2024-09-13 21:49:55.841532968 +0000 UTC m=+0.653743599 container remove 1b0f428d8716caab4d01d65d03e6bfe8d2b1ee9b10683e47d595b5905cd6c57e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, name=centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, release=754, GIT_CLEAN=True, vcs-type=git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T21:49:56.504 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:56 smithi181 podman[56180]: 2024-09-13 21:49:56.128552402 +0000 UTC m=+0.017857967 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:49:56.505 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:56 smithi181 podman[56180]: 2024-09-13 21:49:56.251230151 +0000 UTC m=+0.140535709 container create 2546a79cdb6a0cbfe7b454ce90ba8cc0d797880261583510809c94f52bc6fca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:49:56.833 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:56 smithi154 ceph-mon[65435]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T21:49:56.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:56 smithi181 ceph-mon[51676]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T21:49:56.900 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:56 smithi181 podman[56180]: 2024-09-13 21:49:56.504059708 +0000 UTC m=+0.393365263 container init 2546a79cdb6a0cbfe7b454ce90ba8cc0d797880261583510809c94f52bc6fca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T21:49:56.900 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:56 smithi181 podman[56180]: 2024-09-13 21:49:56.508735886 +0000 UTC m=+0.398041442 container start 2546a79cdb6a0cbfe7b454ce90ba8cc0d797880261583510809c94f52bc6fca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T21:49:56.901 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:56 smithi181 podman[56180]: 2024-09-13 21:49:56.604263403 +0000 UTC m=+0.493568969 container attach 2546a79cdb6a0cbfe7b454ce90ba8cc0d797880261583510809c94f52bc6fca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T21:49:56.901 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:56 smithi181 podman[56180]: 2024-09-13 21:49:56.713233606 +0000 UTC m=+0.602539170 container died 2546a79cdb6a0cbfe7b454ce90ba8cc0d797880261583510809c94f52bc6fca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T21:49:56.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 221749 10.20 MB/sec execute 429 sec latency 43.357 ms 2024-09-13T21:49:57.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:57 smithi154 ceph-mon[65435]: pgmap v189: 97 pgs: 6 active+undersized, 13 peering, 2 stale+active+clean, 18 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.4 MiB/s wr, 403 op/s; 150/1539 objects degraded (9.747%) 2024-09-13T21:49:57.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:57 smithi154 ceph-mon[65435]: Health check failed: Degraded data redundancy: 150/1539 objects degraded (9.747%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:57.869 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:57 smithi181 ceph-mon[51676]: pgmap v189: 97 pgs: 6 active+undersized, 13 peering, 2 stale+active+clean, 18 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.4 MiB/s wr, 403 op/s; 150/1539 objects degraded (9.747%) 2024-09-13T21:49:57.870 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:57 smithi181 ceph-mon[51676]: Health check failed: Degraded data redundancy: 150/1539 objects degraded (9.747%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T21:49:57.870 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:57 smithi181 podman[56203]: 2024-09-13 21:49:57.40979871 +0000 UTC m=+0.688708660 container remove 2546a79cdb6a0cbfe7b454ce90ba8cc0d797880261583510809c94f52bc6fca8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T21:49:57.870 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:57 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-13T21:49:57.870 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:57 smithi181 systemd[1]: Stopped Ceph osd.4 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:49:57.870 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:57 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.4.service: Consumed 3min 48.577s CPU time. 2024-09-13T21:49:57.871 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:57 smithi181 systemd[1]: Starting Ceph osd.4 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:49:57.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 222133 10.19 MB/sec execute 430 sec latency 56.948 ms 2024-09-13T21:49:58.150 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:57 smithi181 podman[56306]: 2024-09-13 21:49:57.771103888 +0000 UTC m=+0.018780894 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:49:58.150 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:57 smithi181 podman[56306]: 2024-09-13 21:49:57.8962375 +0000 UTC m=+0.143914493 container create df6e3e3610c36c1cafed46158b2599b38c0f47170c2ec7d4a0dddcdfaf2d96f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T21:49:58.650 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:58 smithi181 podman[56306]: 2024-09-13 21:49:58.15588241 +0000 UTC m=+0.403559405 container init df6e3e3610c36c1cafed46158b2599b38c0f47170c2ec7d4a0dddcdfaf2d96f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be) 2024-09-13T21:49:58.650 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:58 smithi181 podman[56306]: 2024-09-13 21:49:58.159332436 +0000 UTC m=+0.407009436 container start df6e3e3610c36c1cafed46158b2599b38c0f47170c2ec7d4a0dddcdfaf2d96f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T21:49:58.650 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:58 smithi181 podman[56306]: 2024-09-13 21:49:58.255937724 +0000 UTC m=+0.503614736 container attach df6e3e3610c36c1cafed46158b2599b38c0f47170c2ec7d4a0dddcdfaf2d96f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T21:49:58.650 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:58 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:49:58.650 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:58 smithi181 bash[56306]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:49:58.650 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:58 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:49:58.650 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:58 smithi181 bash[56306]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:49:58.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 222529 10.19 MB/sec execute 431 sec latency 61.173 ms 2024-09-13T21:49:59.635 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:49:59 smithi154 ceph-mon[65435]: pgmap v190: 97 pgs: 8 active+undersized, 13 peering, 20 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 395 op/s; 167/1521 objects degraded (10.980%) 2024-09-13T21:49:59.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:49:59 smithi181 ceph-mon[51676]: pgmap v190: 97 pgs: 8 active+undersized, 13 peering, 20 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 395 op/s; 167/1521 objects degraded (10.980%) 2024-09-13T21:49:59.900 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T21:49:59.901 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 bash[56306]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T21:49:59.901 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T21:49:59.901 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 bash[56306]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T21:49:59.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 222854 10.19 MB/sec execute 432 sec latency 47.898 ms 2024-09-13T21:50:00.370 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T21:50:00.370 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 bash[56306]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T21:50:00.370 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T21:50:00.370 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 bash[56306]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T21:50:00.370 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T21:50:00.370 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 bash[56306]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T21:50:00.370 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T21:50:00.371 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 bash[56306]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T21:50:00.371 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate[56321]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T21:50:00.371 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:49:59 smithi181 bash[56306]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T21:50:00.371 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:00 smithi181 podman[56306]: 2024-09-13 21:50:00.016631026 +0000 UTC m=+2.264308039 container died df6e3e3610c36c1cafed46158b2599b38c0f47170c2ec7d4a0dddcdfaf2d96f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 167/1521 objects degraded (10.980%), 20 pgs degraded 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: osd.4 (root=default,host=smithi181) is down 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: [WRN] PG_DEGRADED: Degraded data redundancy: 167/1521 objects degraded (10.980%), 20 pgs degraded 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 2.5 is active+undersized+degraded, acting [2,0] 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-13T21:50:00.649 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T21:50:00.650 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:00 smithi154 ceph-mon[65435]: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 167/1521 objects degraded (10.980%), 20 pgs degraded 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: osd.4 (root=default,host=smithi181) is down 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: [WRN] PG_DEGRADED: Degraded data redundancy: 167/1521 objects degraded (10.980%), 20 pgs degraded 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 2.5 is active+undersized+degraded, acting [2,0] 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-13T21:50:00.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T21:50:00.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:00 smithi181 ceph-mon[51676]: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-13T21:50:00.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 223268 10.20 MB/sec execute 433 sec latency 46.438 ms 2024-09-13T21:50:01.081 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:00 smithi181 podman[56459]: 2024-09-13 21:50:00.712431854 +0000 UTC m=+0.689159975 container remove df6e3e3610c36c1cafed46158b2599b38c0f47170c2ec7d4a0dddcdfaf2d96f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T21:50:01.082 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:00 smithi181 podman[56519]: 2024-09-13 21:50:00.832153897 +0000 UTC m=+0.018047737 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:50:01.082 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:00 smithi181 podman[56519]: 2024-09-13 21:50:00.955078667 +0000 UTC m=+0.140972498 container create e8ce19a28c45ae3b23987cb3c0f5873249f690f994c123e7e36fa78623bb3198 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be) 2024-09-13T21:50:01.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:01 smithi181 podman[56519]: 2024-09-13 21:50:01.207683609 +0000 UTC m=+0.393577442 container init e8ce19a28c45ae3b23987cb3c0f5873249f690f994c123e7e36fa78623bb3198 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:50:01.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:01 smithi181 podman[56519]: 2024-09-13 21:50:01.21113295 +0000 UTC m=+0.397026784 container start e8ce19a28c45ae3b23987cb3c0f5873249f690f994c123e7e36fa78623bb3198 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be) 2024-09-13T21:50:01.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:01 smithi181 ceph-osd[56539]: -- 172.21.15.181:0/503893591 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5593d8cc01e0 con 0x5593d8c9e000 2024-09-13T21:50:01.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:01 smithi181 bash[56519]: e8ce19a28c45ae3b23987cb3c0f5873249f690f994c123e7e36fa78623bb3198 2024-09-13T21:50:01.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:01 smithi181 systemd[1]: Started Ceph osd.4 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:50:01.654 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:01 smithi154 ceph-mon[65435]: pgmap v191: 97 pgs: 12 active+undersized, 8 peering, 21 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 2.7 MiB/s wr, 283 op/s; 181/1551 objects degraded (11.670%) 2024-09-13T21:50:01.654 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:01.654 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:01.655 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:01 smithi181 ceph-mon[51676]: pgmap v191: 97 pgs: 12 active+undersized, 8 peering, 21 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 2.7 MiB/s wr, 283 op/s; 181/1551 objects degraded (11.670%) 2024-09-13T21:50:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:01.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:01.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 223662 10.20 MB/sec execute 434 sec latency 55.131 ms 2024-09-13T21:50:02.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:02 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 255/1617 objects degraded (15.770%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:02.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:02 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 255/1617 objects degraded (15.770%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:02.900 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:02 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:50:02.862+0000 7ff27daa4640 -1 Falling back to public interface 2024-09-13T21:50:02.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 224018 10.19 MB/sec execute 435 sec latency 56.507 ms 2024-09-13T21:50:03.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:03 smithi181 ceph-mon[51676]: pgmap v192: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 255/1617 objects degraded (15.770%) 2024-09-13T21:50:03.737 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:03 smithi154 ceph-mon[65435]: pgmap v192: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 255/1617 objects degraded (15.770%) 2024-09-13T21:50:03.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 224229 10.18 MB/sec execute 436 sec latency 60.558 ms 2024-09-13T21:50:04.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 224569 10.18 MB/sec execute 437 sec latency 49.181 ms 2024-09-13T21:50:05.795 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:05 smithi154 ceph-mon[65435]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 435 op/s; 260/1632 objects degraded (15.931%) 2024-09-13T21:50:05.795 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:05 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:05.795 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:05 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:05.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:05 smithi181 ceph-mon[51676]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 435 op/s; 260/1632 objects degraded (15.931%) 2024-09-13T21:50:05.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:05 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:05.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:05 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:05.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 224790 10.17 MB/sec execute 438 sec latency 71.836 ms 2024-09-13T21:50:06.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 225067 10.17 MB/sec execute 439 sec latency 74.304 ms 2024-09-13T21:50:07.357 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:07.357 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:07.357 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:07 smithi154 ceph-mon[65435]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.6 MiB/s wr, 578 op/s; 263/1635 objects degraded (16.086%) 2024-09-13T21:50:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:07 smithi181 ceph-mon[51676]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.6 MiB/s wr, 578 op/s; 263/1635 objects degraded (16.086%) 2024-09-13T21:50:07.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 225316 10.16 MB/sec execute 440 sec latency 49.137 ms 2024-09-13T21:50:08.653 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:08 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 258/1617 objects degraded (15.955%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:08.679 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:08 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 258/1617 objects degraded (15.955%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:08.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 225483 10.15 MB/sec execute 441 sec latency 119.230 ms 2024-09-13T21:50:09.719 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:09 smithi154 ceph-mon[65435]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.3 MiB/s wr, 428 op/s; 258/1617 objects degraded (15.955%) 2024-09-13T21:50:09.900 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:09 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:50:09.828+0000 7ff27daa4640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-13T21:50:09.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:09 smithi181 ceph-mon[51676]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.3 MiB/s wr, 428 op/s; 258/1617 objects degraded (15.955%) 2024-09-13T21:50:09.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 225863 10.14 MB/sec execute 442 sec latency 142.501 ms 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:10.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:10.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:10.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T21:50:10.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T21:50:10.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T21:50:10.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:10.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:50:10.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 226173 10.13 MB/sec execute 443 sec latency 43.635 ms 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:11.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:11.072 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T21:50:11.072 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T21:50:11.072 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T21:50:11.072 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:11.072 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:50:11.301 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:50:11.400 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:11 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:50:11.069+0000 7ff27daa4640 -1 osd.4 113 log_to_monitors true 2024-09-13T21:50:11.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:11 smithi154 ceph-mon[65435]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 467 op/s; 261/1608 objects degraded (16.231%) 2024-09-13T21:50:11.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:11 smithi154 ceph-mon[65435]: from='osd.4 [v2:172.21.15.181:6800/3582768641,v1:172.21.15.181:6801/3582768641]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:50:11.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:11 smithi154 ceph-mon[65435]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:50:11.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 226550 10.14 MB/sec execute 444 sec latency 50.061 ms 2024-09-13T21:50:12.150 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:50:11 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:50:11.746+0000 7ff27584f640 -1 osd.4 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:50:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:11 smithi181 ceph-mon[51676]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 467 op/s; 261/1608 objects degraded (16.231%) 2024-09-13T21:50:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:11 smithi181 ceph-mon[51676]: from='osd.4 [v2:172.21.15.181:6800/3582768641,v1:172.21.15.181:6801/3582768641]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:50:12.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:11 smithi181 ceph-mon[51676]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (7m) 45s ago 15m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (3m) 45s ago 15m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (3m) 6s ago 15m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (7m) 45s ago 15m 52.7M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (9m) 45s ago 9m 37.3M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (9m) 6s ago 9m 17.6M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (6m) 45s ago 17m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (5m) 6s ago 15m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (5m) 45s ago 17m 50.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:50:12.294 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (4m) 6s ago 15m 40.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (9m) 45s ago 9m 279M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (9m) 6s ago 9m 84.7M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (6m) 45s ago 15m 15.4M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (6m) 6s ago 14m 15.2M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (3m) 45s ago 14m 333M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (2m) 45s ago 13m 257M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (101s) 45s ago 13m 201M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (54s) 45s ago 13m 14.2M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (11s) 6s ago 12m 11.2M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (12m) 6s ago 12m 527M 2703M 17.2.0 e1d6a67b021e 96b1af27db5c 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (11m) 6s ago 11m 451M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (11m) 6s ago 11m 692M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:50:12.295 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (5m) 45s ago 14m 66.1M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:50:12.776 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:12 smithi154 ceph-mon[65435]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:12.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:12 smithi154 ceph-mon[65435]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T21:50:12.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:12 smithi154 ceph-mon[65435]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T21:50:12.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:12 smithi154 ceph-mon[65435]: from='osd.4 [v2:172.21.15.181:6800/3582768641,v1:172.21.15.181:6801/3582768641]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:50:12.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:12 smithi154 ceph-mon[65435]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:50:12.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:12 smithi154 ceph-mon[65435]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:12.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 226844 10.14 MB/sec execute 445 sec latency 183.326 ms 2024-09-13T21:50:12.932 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:50:12.932 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:50:12.932 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:50:12.932 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:50:12.932 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:50:12.932 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:50:12.932 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 9 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:50:12.933 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:50:13.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:12 smithi181 ceph-mon[51676]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:13.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:12 smithi181 ceph-mon[51676]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T21:50:13.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:12 smithi181 ceph-mon[51676]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T21:50:13.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:12 smithi181 ceph-mon[51676]: from='osd.4 [v2:172.21.15.181:6800/3582768641,v1:172.21.15.181:6801/3582768641]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:50:13.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:12 smithi181 ceph-mon[51676]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:50:13.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:12 smithi181 ceph-mon[51676]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:50:13.407 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "11/23 daemons upgraded", 2024-09-13T21:50:13.408 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:50:13.408 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:50:13.408 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:50:13.897 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_WARN Degraded data redundancy: 268/1644 objects degraded (16.302%), 28 pgs degraded 2024-09-13T21:50:13.897 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 268/1644 objects degraded (16.302%), 28 pgs degraded 2024-09-13T21:50:13.897 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.5 is active+undersized+degraded, acting [2,0] 2024-09-13T21:50:13.897 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-13T21:50:13.897 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-13T21:50:13.897 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-13T21:50:13.897 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-13T21:50:13.897 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-13T21:50:13.898 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T21:50:13.899 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-13T21:50:13.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 226921 10.12 MB/sec execute 446 sec latency 1161.034 ms 2024-09-13T21:50:14.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:13 smithi154 ceph-mon[65435]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:14.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:13 smithi154 ceph-mon[65435]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 402 op/s; 268/1644 objects degraded (16.302%) 2024-09-13T21:50:14.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:13 smithi154 ceph-mon[65435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:50:14.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:13 smithi154 ceph-mon[65435]: osd.4 [v2:172.21.15.181:6800/3582768641,v1:172.21.15.181:6801/3582768641] boot 2024-09-13T21:50:14.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:13 smithi154 ceph-mon[65435]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T21:50:14.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:14.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:13 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2206036348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:13 smithi181 ceph-mon[51676]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:13 smithi181 ceph-mon[51676]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 402 op/s; 268/1644 objects degraded (16.302%) 2024-09-13T21:50:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:13 smithi181 ceph-mon[51676]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:50:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:13 smithi181 ceph-mon[51676]: osd.4 [v2:172.21.15.181:6800/3582768641,v1:172.21.15.181:6801/3582768641] boot 2024-09-13T21:50:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:13 smithi181 ceph-mon[51676]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T21:50:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T21:50:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:13 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2206036348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:14.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 226954 10.10 MB/sec execute 447 sec latency 2054.729 ms 2024-09-13T21:50:14.997 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:14 smithi154 ceph-mon[65435]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:14.997 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:14 smithi154 ceph-mon[65435]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T21:50:14.997 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:14 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/3390245181' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:50:14.997 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:14 smithi154 ceph-mon[65435]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T21:50:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:14 smithi181 ceph-mon[51676]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:14 smithi181 ceph-mon[51676]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T21:50:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:14 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/3390245181' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:50:15.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:14 smithi181 ceph-mon[51676]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T21:50:15.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:15 smithi154 ceph-mon[65435]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.0 MiB/s wr, 294 op/s; 270/1653 objects degraded (16.334%) 2024-09-13T21:50:15.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:15 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 270/1653 objects degraded (16.334%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:15.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:15 smithi154 ceph-mon[65435]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T21:50:15.928 INFO:teuthology.orchestra.run.smithi154.stdout: 5 227284 10.10 MB/sec execute 448 sec latency 60.819 ms 2024-09-13T21:50:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:15 smithi181 ceph-mon[51676]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.0 MiB/s wr, 294 op/s; 270/1653 objects degraded (16.334%) 2024-09-13T21:50:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:15 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 270/1653 objects degraded (16.334%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:15 smithi181 ceph-mon[51676]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T21:50:16.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 227562 10.09 MB/sec execute 449 sec latency 38.334 ms 2024-09-13T21:50:17.813 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:17 smithi154 ceph-mon[65435]: pgmap v204: 97 pgs: 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized, 3 active+undersized+degraded, 73 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.8 MiB/s wr, 328 op/s; 134/1539 objects degraded (8.707%); 3.4 MiB/s, 52 keys/s, 57 objects/s recovering 2024-09-13T21:50:17.813 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:17 smithi154 ceph-mon[65435]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T21:50:17.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:17 smithi181 ceph-mon[51676]: pgmap v204: 97 pgs: 3 peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized, 3 active+undersized+degraded, 73 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.8 MiB/s wr, 328 op/s; 134/1539 objects degraded (8.707%); 3.4 MiB/s, 52 keys/s, 57 objects/s recovering 2024-09-13T21:50:17.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:17 smithi181 ceph-mon[51676]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T21:50:17.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 227871 10.09 MB/sec execute 450 sec latency 485.726 ms 2024-09-13T21:50:18.831 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:18 smithi154 ceph-mon[65435]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T21:50:18.831 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:18 smithi154 ceph-mon[65435]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T21:50:18.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:18 smithi181 ceph-mon[51676]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T21:50:18.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:18 smithi181 ceph-mon[51676]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T21:50:18.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 228247 10.08 MB/sec execute 451 sec latency 985.818 ms 2024-09-13T21:50:19.859 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:19 smithi154 ceph-mon[65435]: pgmap v207: 97 pgs: 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 566 op/s; 114/1533 objects degraded (7.436%); 3.5 MiB/s, 52 keys/s, 62 objects/s recovering 2024-09-13T21:50:19.859 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:19 smithi154 ceph-mon[65435]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T21:50:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:19 smithi181 ceph-mon[51676]: pgmap v207: 97 pgs: 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 566 op/s; 114/1533 objects degraded (7.436%); 3.5 MiB/s, 52 keys/s, 62 objects/s recovering 2024-09-13T21:50:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:19 smithi181 ceph-mon[51676]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T21:50:19.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 228593 10.08 MB/sec execute 452 sec latency 810.627 ms 2024-09-13T21:50:20.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:20 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 114/1533 objects degraded (7.436%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:20.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:20 smithi154 ceph-mon[65435]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T21:50:20.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 229005 10.08 MB/sec execute 453 sec latency 999.415 ms 2024-09-13T21:50:21.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:20 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 114/1533 objects degraded (7.436%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:21.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:20 smithi181 ceph-mon[51676]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T21:50:21.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:21 smithi154 ceph-mon[65435]: pgmap v210: 97 pgs: 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.0 MiB/s wr, 295 op/s; 114/1542 objects degraded (7.393%); 82 KiB/s, 5 objects/s recovering 2024-09-13T21:50:21.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 229425 10.09 MB/sec execute 454 sec latency 33.215 ms 2024-09-13T21:50:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:21 smithi181 ceph-mon[51676]: pgmap v210: 97 pgs: 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.0 MiB/s wr, 295 op/s; 114/1542 objects degraded (7.393%); 82 KiB/s, 5 objects/s recovering 2024-09-13T21:50:22.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 229830 10.10 MB/sec execute 455 sec latency 47.459 ms 2024-09-13T21:50:23.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:23 smithi154 ceph-mon[65435]: pgmap v212: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 455 op/s; 75 KiB/s, 24 objects/s recovering 2024-09-13T21:50:23.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:23 smithi154 ceph-mon[65435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 114/1542 objects degraded (7.393%), 5 pgs degraded) 2024-09-13T21:50:23.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:23 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:50:23.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 230150 10.09 MB/sec execute 456 sec latency 54.809 ms 2024-09-13T21:50:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:23 smithi181 ceph-mon[51676]: pgmap v212: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 455 op/s; 75 KiB/s, 24 objects/s recovering 2024-09-13T21:50:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:23 smithi181 ceph-mon[51676]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 114/1542 objects degraded (7.393%), 5 pgs degraded) 2024-09-13T21:50:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:23 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:50:24.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 230514 10.09 MB/sec execute 457 sec latency 43.253 ms 2024-09-13T21:50:25.713 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:25 smithi154 ceph-mon[65435]: pgmap v213: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 382 op/s; 61 KiB/s, 19 objects/s recovering 2024-09-13T21:50:25.714 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T21:50:25.714 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:25.714 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:50:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:25 smithi181 ceph-mon[51676]: pgmap v213: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 382 op/s; 61 KiB/s, 19 objects/s recovering 2024-09-13T21:50:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T21:50:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:25.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:50:25.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 230815 10.08 MB/sec execute 458 sec latency 54.364 ms 2024-09-13T21:50:26.724 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:26 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T21:50:26.724 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:26 smithi154 ceph-mon[65435]: Upgrade: osd.5 is safe to restart 2024-09-13T21:50:26.724 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:26 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:26.724 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:26 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:50:26.724 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:26 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:26.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:26 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T21:50:26.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:26 smithi181 ceph-mon[51676]: Upgrade: osd.5 is safe to restart 2024-09-13T21:50:26.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:26 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:26.915 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:26 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T21:50:26.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:26 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:26.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 231074 10.08 MB/sec execute 459 sec latency 43.350 ms 2024-09-13T21:50:27.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:27 smithi154 ceph-mon[65435]: Upgrade: Updating osd.5 2024-09-13T21:50:27.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:27 smithi154 ceph-mon[65435]: Deploying daemon osd.5 on smithi181 2024-09-13T21:50:27.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:27 smithi154 ceph-mon[65435]: pgmap v214: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 6.4 MiB/s wr, 605 op/s; 46 KiB/s, 15 objects/s recovering 2024-09-13T21:50:27.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:27 smithi181 ceph-mon[51676]: Upgrade: Updating osd.5 2024-09-13T21:50:27.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:27 smithi181 ceph-mon[51676]: Deploying daemon osd.5 on smithi181 2024-09-13T21:50:27.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:27 smithi181 ceph-mon[51676]: pgmap v214: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 6.4 MiB/s wr, 605 op/s; 46 KiB/s, 15 objects/s recovering 2024-09-13T21:50:27.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 231374 10.08 MB/sec execute 460 sec latency 45.061 ms 2024-09-13T21:50:28.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 231670 10.07 MB/sec execute 461 sec latency 44.336 ms 2024-09-13T21:50:29.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:29 smithi154 ceph-mon[65435]: pgmap v215: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 557 op/s; 41 KiB/s, 13 objects/s recovering 2024-09-13T21:50:29.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:29 smithi181 ceph-mon[51676]: pgmap v215: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 557 op/s; 41 KiB/s, 13 objects/s recovering 2024-09-13T21:50:29.778 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:29 smithi181 systemd[1]: Stopping Ceph osd.5 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:50:29.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 232003 10.06 MB/sec execute 462 sec latency 53.536 ms 2024-09-13T21:50:30.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[33013]: 2024-09-13T21:50:29.813+0000 7f21d38b3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:50:30.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[33013]: 2024-09-13T21:50:29.813+0000 7f21d38b3700 -1 osd.5 125 *** Got signal Terminated *** 2024-09-13T21:50:30.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[33013]: 2024-09-13T21:50:29.813+0000 7f21d38b3700 -1 osd.5 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:50:30.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:30 smithi154 ceph-mon[65435]: osd.5 marked itself down and dead 2024-09-13T21:50:30.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:30 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:30.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 232212 10.05 MB/sec execute 463 sec latency 205.019 ms 2024-09-13T21:50:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:30 smithi181 ceph-mon[51676]: osd.5 marked itself down and dead 2024-09-13T21:50:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:30 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:31.606 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:31 smithi181 podman[58337]: 2024-09-13 21:50:31.309257776 +0000 UTC m=+1.610976847 container died 96b1af27db5c60c89c508ab81cd6988e951640a8597f59ece43acf8803dd7070 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, name=centos-stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, version=8, release=754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0) 2024-09-13T21:50:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:31 smithi181 ceph-mon[51676]: pgmap v216: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 488 op/s; 37 KiB/s, 11 objects/s recovering 2024-09-13T21:50:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:31 smithi181 ceph-mon[51676]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:50:31.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:31 smithi181 ceph-mon[51676]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T21:50:31.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:31 smithi181 podman[58337]: 2024-09-13 21:50:31.696877202 +0000 UTC m=+1.998596280 container cleanup 96b1af27db5c60c89c508ab81cd6988e951640a8597f59ece43acf8803dd7070 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, vcs-type=git, RELEASE=HEAD, release=754, architecture=x86_64, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, distribution-scope=public, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-09-13T21:50:31.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:31 smithi181 bash[58337]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5 2024-09-13T21:50:31.914 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:31 smithi154 ceph-mon[65435]: pgmap v216: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 488 op/s; 37 KiB/s, 11 objects/s recovering 2024-09-13T21:50:31.914 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:31 smithi154 ceph-mon[65435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:50:31.914 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:31 smithi154 ceph-mon[65435]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T21:50:31.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 232321 10.03 MB/sec execute 464 sec latency 979.435 ms 2024-09-13T21:50:32.358 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:31 smithi181 podman[58366]: 2024-09-13 21:50:31.990429874 +0000 UTC m=+0.681917902 container remove 96b1af27db5c60c89c508ab81cd6988e951640a8597f59ece43acf8803dd7070 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, vendor=Red Hat, Inc., architecture=x86_64, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T21:50:32.650 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:32 smithi181 podman[58450]: 2024-09-13 21:50:32.261054376 +0000 UTC m=+0.019962825 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:50:32.650 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:32 smithi181 podman[58450]: 2024-09-13 21:50:32.366429708 +0000 UTC m=+0.125338143 container create 66377b2e9d23f49c5b7966a235d56a20bcc47e3da1f8ec6c4f3aa8269a68dcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD) 2024-09-13T21:50:32.920 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:32 smithi154 ceph-mon[65435]: osdmap e127: 8 total, 7 up, 8 in 2024-09-13T21:50:32.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 232850 10.05 MB/sec execute 465 sec latency 53.234 ms 2024-09-13T21:50:33.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:32 smithi181 ceph-mon[51676]: osdmap e127: 8 total, 7 up, 8 in 2024-09-13T21:50:33.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:32 smithi181 podman[58450]: 2024-09-13 21:50:32.695503731 +0000 UTC m=+0.454412163 container init 66377b2e9d23f49c5b7966a235d56a20bcc47e3da1f8ec6c4f3aa8269a68dcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2) 2024-09-13T21:50:33.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:32 smithi181 podman[58450]: 2024-09-13 21:50:32.699175767 +0000 UTC m=+0.458084205 container start 66377b2e9d23f49c5b7966a235d56a20bcc47e3da1f8ec6c4f3aa8269a68dcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-13T21:50:33.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:32 smithi181 podman[58450]: 2024-09-13 21:50:32.787239926 +0000 UTC m=+0.546148372 container attach 66377b2e9d23f49c5b7966a235d56a20bcc47e3da1f8ec6c4f3aa8269a68dcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-13T21:50:33.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:32 smithi181 podman[58450]: 2024-09-13 21:50:32.922477821 +0000 UTC m=+0.681386260 container died 66377b2e9d23f49c5b7966a235d56a20bcc47e3da1f8ec6c4f3aa8269a68dcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:50:33.796 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:33 smithi181 ceph-mon[51676]: pgmap v219: 97 pgs: 1 active+undersized, 15 peering, 4 stale+active+clean, 3 active+undersized+degraded, 74 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.1 MiB/s wr, 447 op/s; 24/1608 objects degraded (1.493%) 2024-09-13T21:50:33.797 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:33 smithi181 ceph-mon[51676]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T21:50:33.797 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:33 smithi181 ceph-mon[51676]: Health check failed: Degraded data redundancy: 24/1608 objects degraded (1.493%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:33.797 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:33 smithi181 podman[58473]: 2024-09-13 21:50:33.535505103 +0000 UTC m=+0.604310199 container remove 66377b2e9d23f49c5b7966a235d56a20bcc47e3da1f8ec6c4f3aa8269a68dcfa (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, ceph=True) 2024-09-13T21:50:33.797 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:33 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-13T21:50:33.797 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:33 smithi181 systemd[1]: Stopped Ceph osd.5 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:50:33.797 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:33 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 27.709s CPU time. 2024-09-13T21:50:33.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 233410 10.06 MB/sec execute 466 sec latency 41.698 ms 2024-09-13T21:50:33.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:33 smithi154 ceph-mon[65435]: pgmap v219: 97 pgs: 1 active+undersized, 15 peering, 4 stale+active+clean, 3 active+undersized+degraded, 74 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.1 MiB/s wr, 447 op/s; 24/1608 objects degraded (1.493%) 2024-09-13T21:50:33.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:33 smithi154 ceph-mon[65435]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T21:50:33.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:33 smithi154 ceph-mon[65435]: Health check failed: Degraded data redundancy: 24/1608 objects degraded (1.493%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:34.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:33 smithi181 systemd[1]: Starting Ceph osd.5 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:50:34.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 podman[58572]: 2024-09-13 21:50:33.923751063 +0000 UTC m=+0.020274768 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:50:34.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 podman[58572]: 2024-09-13 21:50:34.035156362 +0000 UTC m=+0.131680045 container create 20debf2aa910a677fefd402763c19e30ed09f3b498ff25435a167edee9f11c4e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T21:50:34.424 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 podman[58572]: 2024-09-13 21:50:34.263118133 +0000 UTC m=+0.359641817 container init 20debf2aa910a677fefd402763c19e30ed09f3b498ff25435a167edee9f11c4e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-13T21:50:34.424 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 podman[58572]: 2024-09-13 21:50:34.267296416 +0000 UTC m=+0.363820103 container start 20debf2aa910a677fefd402763c19e30ed09f3b498ff25435a167edee9f11c4e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:50:34.424 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 podman[58572]: 2024-09-13 21:50:34.346988643 +0000 UTC m=+0.443512337 container attach 20debf2aa910a677fefd402763c19e30ed09f3b498ff25435a167edee9f11c4e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T21:50:34.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:50:34.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 bash[58572]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:50:34.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:50:34.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:34 smithi181 bash[58572]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:50:34.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 233828 10.06 MB/sec execute 467 sec latency 29.020 ms 2024-09-13T21:50:35.690 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:35 smithi154 ceph-mon[65435]: pgmap v220: 97 pgs: 3 active+undersized, 15 peering, 3 stale+active+clean, 5 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.4 MiB/s wr, 260 op/s; 31/1608 objects degraded (1.928%) 2024-09-13T21:50:35.796 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:35 smithi181 ceph-mon[51676]: pgmap v220: 97 pgs: 3 active+undersized, 15 peering, 3 stale+active+clean, 5 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.4 MiB/s wr, 260 op/s; 31/1608 objects degraded (1.928%) 2024-09-13T21:50:35.796 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T21:50:35.796 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 bash[58572]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T21:50:35.796 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T21:50:35.796 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 bash[58572]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T21:50:35.796 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T21:50:35.796 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 bash[58572]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T21:50:35.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 234229 10.07 MB/sec execute 468 sec latency 38.017 ms 2024-09-13T21:50:36.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T21:50:36.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 bash[58572]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T21:50:36.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T21:50:36.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 bash[58572]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T21:50:36.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T21:50:36.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 bash[58572]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T21:50:36.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate[58586]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T21:50:36.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 bash[58572]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T21:50:36.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:35 smithi181 podman[58572]: 2024-09-13 21:50:35.826548711 +0000 UTC m=+1.923072409 container died 20debf2aa910a677fefd402763c19e30ed09f3b498ff25435a167edee9f11c4e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-13T21:50:36.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 234619 10.07 MB/sec execute 469 sec latency 37.243 ms 2024-09-13T21:50:36.967 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:36 smithi154 ceph-mon[65435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-13T21:50:36.995 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:36 smithi181 ceph-mon[51676]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-13T21:50:36.996 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:36 smithi181 podman[58723]: 2024-09-13 21:50:36.694268649 +0000 UTC m=+0.861286823 container remove 20debf2aa910a677fefd402763c19e30ed09f3b498ff25435a167edee9f11c4e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-13T21:50:36.996 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:36 smithi181 podman[58780]: 2024-09-13 21:50:36.81358447 +0000 UTC m=+0.017059104 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:50:36.996 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:36 smithi181 podman[58780]: 2024-09-13 21:50:36.995324883 +0000 UTC m=+0.198799502 container create 80819532f2ce3c06e988b8fe7fb75cfa5c0e28e039f8353db7e8e1069c6b5227 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T21:50:37.258 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:37 smithi181 podman[58780]: 2024-09-13 21:50:37.231606374 +0000 UTC m=+0.435080992 container init 80819532f2ce3c06e988b8fe7fb75cfa5c0e28e039f8353db7e8e1069c6b5227 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-13T21:50:37.258 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:37 smithi181 podman[58780]: 2024-09-13 21:50:37.234938523 +0000 UTC m=+0.438413142 container start 80819532f2ce3c06e988b8fe7fb75cfa5c0e28e039f8353db7e8e1069c6b5227 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T21:50:37.650 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:37 smithi181 ceph-osd[58799]: -- 172.21.15.181:0/2809460270 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55939a9ee1e0 con 0x559399db8000 2024-09-13T21:50:37.650 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:37 smithi181 bash[58780]: 80819532f2ce3c06e988b8fe7fb75cfa5c0e28e039f8353db7e8e1069c6b5227 2024-09-13T21:50:37.650 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:37 smithi181 systemd[1]: Started Ceph osd.5 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:50:37.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 234957 10.07 MB/sec execute 470 sec latency 50.441 ms 2024-09-13T21:50:37.977 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:37 smithi154 ceph-mon[65435]: pgmap v221: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.2 MiB/s wr, 495 op/s; 189/1674 objects degraded (11.290%) 2024-09-13T21:50:37.977 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:37 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:37.977 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:37 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:37.978 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:37 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:38.090 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:37 smithi181 ceph-mon[51676]: pgmap v221: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.2 MiB/s wr, 495 op/s; 189/1674 objects degraded (11.290%) 2024-09-13T21:50:38.090 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:37 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:38.091 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:37 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:38.091 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:37 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:38.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:38 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 187/1701 objects degraded (10.994%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:38.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:38 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:50:38.634+0000 7fae5c103640 -1 Falling back to public interface 2024-09-13T21:50:38.924 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:38 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 187/1701 objects degraded (10.994%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:38.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 235305 10.06 MB/sec execute 471 sec latency 46.429 ms 2024-09-13T21:50:39.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 235702 10.05 MB/sec execute 472 sec latency 52.477 ms 2024-09-13T21:50:39.938 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:39 smithi154 ceph-mon[65435]: pgmap v222: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.0 MiB/s wr, 637 op/s; 187/1701 objects degraded (10.994%) 2024-09-13T21:50:40.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:39 smithi181 ceph-mon[51676]: pgmap v222: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.0 MiB/s wr, 637 op/s; 187/1701 objects degraded (10.994%) 2024-09-13T21:50:40.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 235990 10.05 MB/sec execute 473 sec latency 43.802 ms 2024-09-13T21:50:41.731 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:41.731 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:50:41.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:41 smithi154 ceph-mon[65435]: pgmap v223: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 544 op/s; 187/1710 objects degraded (10.936%) 2024-09-13T21:50:41.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:41.732 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:50:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:41 smithi181 ceph-mon[51676]: pgmap v223: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 544 op/s; 187/1710 objects degraded (10.936%) 2024-09-13T21:50:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:41.933 INFO:teuthology.orchestra.run.smithi154.stdout: 5 236277 10.05 MB/sec execute 474 sec latency 62.247 ms 2024-09-13T21:50:42.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 236594 10.05 MB/sec execute 475 sec latency 83.220 ms 2024-09-13T21:50:43.007 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:42 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:43.007 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:42 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:43.024 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:42 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:43.024 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:42 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:43.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:43 smithi154 ceph-mon[65435]: pgmap v224: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.9 MiB/s wr, 610 op/s; 176/1632 objects degraded (10.784%) 2024-09-13T21:50:43.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 236946 10.05 MB/sec execute 476 sec latency 54.415 ms 2024-09-13T21:50:44.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:43 smithi181 ceph-mon[51676]: pgmap v224: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.9 MiB/s wr, 610 op/s; 176/1632 objects degraded (10.784%) 2024-09-13T21:50:44.468 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:50:44.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 237274 10.04 MB/sec execute 477 sec latency 44.302 ms 2024-09-13T21:50:45.057 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:44 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 176/1629 objects degraded (10.804%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:45.104 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:44 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 176/1629 objects degraded (10.804%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:45.449 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:50:45.449 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (7m) 78s ago 16m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:50:45.449 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (4m) 78s ago 16m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:50:45.449 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (4m) 4s ago 15m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:50:45.449 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (7m) 78s ago 16m 52.7M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:50:45.449 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (10m) 78s ago 10m 37.3M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:50:45.449 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (10m) 4s ago 10m 17.7M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (6m) 78s ago 17m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (6m) 4s ago 15m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (5m) 78s ago 17m 50.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (5m) 4s ago 15m 42.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (10m) 78s ago 10m 279M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (10m) 4s ago 10m 84.7M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (7m) 78s ago 16m 15.4M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (7m) 4s ago 15m 15.1M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (3m) 78s ago 14m 333M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (3m) 78s ago 14m 257M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (2m) 78s ago 13m 201M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (87s) 78s ago 13m 14.2M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (44s) 4s ago 13m 258M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:50:45.450 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (8s) 4s ago 12m 14.4M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 80819532f2ce 2024-09-13T21:50:45.451 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (12m) 4s ago 12m 480M 2703M 17.2.0 e1d6a67b021e 9685c71ddee8 2024-09-13T21:50:45.451 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (12m) 4s ago 12m 740M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:50:45.451 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (6m) 78s ago 15m 66.1M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:50:45.929 INFO:teuthology.orchestra.run.smithi154.stdout: 5 237527 10.04 MB/sec execute 478 sec latency 51.694 ms 2024-09-13T21:50:46.039 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:50:46.040 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:50:46.041 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:50:46.041 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T21:50:46.041 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 9 2024-09-13T21:50:46.041 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:50:46.041 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:50:46.061 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:45 smithi154 ceph-mon[65435]: from='client.44307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:46.061 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:45 smithi154 ceph-mon[65435]: pgmap v225: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.1 MiB/s wr, 543 op/s; 176/1629 objects degraded (10.804%) 2024-09-13T21:50:46.061 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:46.062 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:46.062 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:45 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:45 smithi181 ceph-mon[51676]: from='client.44307 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:45 smithi181 ceph-mon[51676]: pgmap v225: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.1 MiB/s wr, 543 op/s; 176/1629 objects degraded (10.804%) 2024-09-13T21:50:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:45 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:50:46.617 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:50:46.617 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:50:46.617 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:50:46.617 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:50:46.617 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:50:46.618 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:50:46.618 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:50:46.618 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:50:46.618 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:50:46.618 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "12/23 daemons upgraded", 2024-09-13T21:50:46.618 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:50:46.618 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:50:46.618 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:50:46.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 237778 10.04 MB/sec execute 479 sec latency 51.504 ms 2024-09-13T21:50:47.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:47.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T21:50:47.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:46 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2375280318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:47.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:47 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:50:47.032+0000 7fae5c103640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-13T21:50:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:50:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:50:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T21:50:47.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:46 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2375280318' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:50:47.201 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 175/1626 objects degraded (10.763%), 23 pgs degraded 2024-09-13T21:50:47.201 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T21:50:47.201 INFO:teuthology.orchestra.run.smithi154.stdout: osd.5 (root=default,host=smithi181) is down 2024-09-13T21:50:47.201 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 175/1626 objects degraded (10.763%), 23 pgs degraded 2024-09-13T21:50:47.201 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-13T21:50:47.201 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-13T21:50:47.201 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-13T21:50:47.201 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-13T21:50:47.202 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-13T21:50:47.203 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-13T21:50:47.203 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-13T21:50:47.203 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-13T21:50:47.203 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-13T21:50:47.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 238021 10.03 MB/sec execute 480 sec latency 86.297 ms 2024-09-13T21:50:47.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:47 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T21:50:47.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:47 smithi154 ceph-mon[65435]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T21:50:47.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:47 smithi154 ceph-mon[65435]: pgmap v226: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.4 MiB/s wr, 649 op/s; 175/1626 objects degraded (10.763%) 2024-09-13T21:50:47.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:47 smithi154 ceph-mon[65435]: from='client.44323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:47.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:47 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/1157982675' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:50:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:47 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T21:50:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:47 smithi181 ceph-mon[51676]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T21:50:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:47 smithi181 ceph-mon[51676]: pgmap v226: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.4 MiB/s wr, 649 op/s; 175/1626 objects degraded (10.763%) 2024-09-13T21:50:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:47 smithi181 ceph-mon[51676]: from='client.44323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:50:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:47 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/1157982675' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:50:48.650 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:48 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:50:48.208+0000 7fae5c103640 -1 osd.5 125 log_to_monitors true 2024-09-13T21:50:48.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 238337 10.03 MB/sec execute 481 sec latency 78.500 ms 2024-09-13T21:50:48.964 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:48 smithi154 ceph-mon[65435]: from='osd.5 [v2:172.21.15.181:6808/4238838757,v1:172.21.15.181:6809/4238838757]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:48.964 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:48 smithi154 ceph-mon[65435]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:49.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:48 smithi181 ceph-mon[51676]: from='osd.5 [v2:172.21.15.181:6808/4238838757,v1:172.21.15.181:6809/4238838757]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:49.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:48 smithi181 ceph-mon[51676]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T21:50:49.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:50:48 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:50:48.900+0000 7fae53eae640 -1 osd.5 125 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:50:49.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 238629 10.03 MB/sec execute 482 sec latency 62.605 ms 2024-09-13T21:50:50.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:49 smithi154 ceph-mon[65435]: pgmap v227: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 491 op/s; 173/1647 objects degraded (10.504%) 2024-09-13T21:50:50.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:49 smithi154 ceph-mon[65435]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T21:50:50.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:49 smithi154 ceph-mon[65435]: from='osd.5 [v2:172.21.15.181:6808/4238838757,v1:172.21.15.181:6809/4238838757]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:50:50.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:49 smithi154 ceph-mon[65435]: osdmap e128: 8 total, 7 up, 8 in 2024-09-13T21:50:50.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:49 smithi154 ceph-mon[65435]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:50:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:49 smithi181 ceph-mon[51676]: pgmap v227: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 491 op/s; 173/1647 objects degraded (10.504%) 2024-09-13T21:50:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:49 smithi181 ceph-mon[51676]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T21:50:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:49 smithi181 ceph-mon[51676]: from='osd.5 [v2:172.21.15.181:6808/4238838757,v1:172.21.15.181:6809/4238838757]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:50:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:49 smithi181 ceph-mon[51676]: osdmap e128: 8 total, 7 up, 8 in 2024-09-13T21:50:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:49 smithi181 ceph-mon[51676]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:50:50.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 238629 10.01 MB/sec execute 483 sec latency 1062.672 ms 2024-09-13T21:50:51.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:50 smithi181 ceph-mon[51676]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:50:51.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:50 smithi181 ceph-mon[51676]: osd.5 [v2:172.21.15.181:6808/4238838757,v1:172.21.15.181:6809/4238838757] boot 2024-09-13T21:50:51.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:50 smithi181 ceph-mon[51676]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T21:50:51.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:51.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:50 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 173/1647 objects degraded (10.504%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:51.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:50 smithi181 ceph-mon[51676]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T21:50:51.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:50 smithi154 ceph-mon[65435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:50:51.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:50 smithi154 ceph-mon[65435]: osd.5 [v2:172.21.15.181:6808/4238838757,v1:172.21.15.181:6809/4238838757] boot 2024-09-13T21:50:51.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:50 smithi154 ceph-mon[65435]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T21:50:51.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T21:50:51.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:50 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 173/1647 objects degraded (10.504%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:50:51.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:50 smithi154 ceph-mon[65435]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T21:50:51.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 238673 9.99 MB/sec execute 484 sec latency 1909.201 ms 2024-09-13T21:50:52.004 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:51 smithi154 ceph-mon[65435]: pgmap v230: 97 pgs: 2 peering, 8 active+undersized, 21 active+undersized+degraded, 66 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.1 MiB/s wr, 316 op/s; 158/1647 objects degraded (9.593%) 2024-09-13T21:50:52.004 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:51 smithi154 ceph-mon[65435]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T21:50:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:51 smithi181 ceph-mon[51676]: pgmap v230: 97 pgs: 2 peering, 8 active+undersized, 21 active+undersized+degraded, 66 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.1 MiB/s wr, 316 op/s; 158/1647 objects degraded (9.593%) 2024-09-13T21:50:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:51 smithi181 ceph-mon[51676]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T21:50:52.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 238871 9.97 MB/sec execute 485 sec latency 305.954 ms 2024-09-13T21:50:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:53 smithi181 ceph-mon[51676]: pgmap v233: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 5 peering, 4 active+undersized, 9 active+undersized+degraded, 72 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s wr, 397 op/s; 83/1578 objects degraded (5.260%); 22 B/s, 0 objects/s recovering 2024-09-13T21:50:53.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:53 smithi181 ceph-mon[51676]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T21:50:53.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 238922 9.95 MB/sec execute 486 sec latency 1033.511 ms 2024-09-13T21:50:53.949 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:53 smithi154 ceph-mon[65435]: pgmap v233: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 5 peering, 4 active+undersized, 9 active+undersized+degraded, 72 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s wr, 397 op/s; 83/1578 objects degraded (5.260%); 22 B/s, 0 objects/s recovering 2024-09-13T21:50:53.949 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:53 smithi154 ceph-mon[65435]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T21:50:54.726 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:54 smithi154 ceph-mon[65435]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T21:50:54.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 239171 9.94 MB/sec execute 487 sec latency 100.389 ms 2024-09-13T21:50:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:54 smithi181 ceph-mon[51676]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T21:50:55.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:55 smithi154 ceph-mon[65435]: pgmap v236: 97 pgs: 5 remapped+peering, 5 peering, 87 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.3 MiB/s wr, 448 op/s; 947 KiB/s, 23 objects/s recovering 2024-09-13T21:50:55.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:55 smithi154 ceph-mon[65435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 83/1578 objects degraded (5.260%), 11 pgs degraded) 2024-09-13T21:50:55.777 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:55 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:50:55.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:55.778 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:50:55.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 239368 9.92 MB/sec execute 488 sec latency 83.607 ms 2024-09-13T21:50:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:55 smithi181 ceph-mon[51676]: pgmap v236: 97 pgs: 5 remapped+peering, 5 peering, 87 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.3 MiB/s wr, 448 op/s; 947 KiB/s, 23 objects/s recovering 2024-09-13T21:50:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:55 smithi181 ceph-mon[51676]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 83/1578 objects degraded (5.260%), 11 pgs degraded) 2024-09-13T21:50:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:55 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:50:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:50:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:50:56.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 239623 9.92 MB/sec execute 489 sec latency 92.688 ms 2024-09-13T21:50:57.804 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:57 smithi154 ceph-mon[65435]: pgmap v237: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.4 MiB/s wr, 426 op/s; 2.3 MiB/s, 40 objects/s recovering 2024-09-13T21:50:57.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 239867 9.92 MB/sec execute 490 sec latency 83.359 ms 2024-09-13T21:50:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:57 smithi181 ceph-mon[51676]: pgmap v237: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.4 MiB/s wr, 426 op/s; 2.3 MiB/s, 40 objects/s recovering 2024-09-13T21:50:58.933 INFO:teuthology.orchestra.run.smithi154.stdout: 5 240138 9.91 MB/sec execute 491 sec latency 63.707 ms 2024-09-13T21:50:59.882 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:50:59 smithi154 ceph-mon[65435]: pgmap v238: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 2.1 MiB/s wr, 299 op/s; 1.9 MiB/s, 34 objects/s recovering 2024-09-13T21:50:59.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 240569 9.92 MB/sec execute 492 sec latency 53.264 ms 2024-09-13T21:51:00.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:50:59 smithi181 ceph-mon[51676]: pgmap v238: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 2.1 MiB/s wr, 299 op/s; 1.9 MiB/s, 34 objects/s recovering 2024-09-13T21:51:00.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 240926 9.92 MB/sec execute 493 sec latency 62.648 ms 2024-09-13T21:51:01.660 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:01 smithi154 ceph-mon[65435]: pgmap v239: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.8 MiB/s wr, 185 op/s; 1.7 MiB/s, 29 objects/s recovering 2024-09-13T21:51:01.660 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:01 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T21:51:01.750 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:01 smithi181 ceph-mon[51676]: pgmap v239: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.8 MiB/s wr, 185 op/s; 1.7 MiB/s, 29 objects/s recovering 2024-09-13T21:51:01.751 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:01 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T21:51:01.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 241205 9.92 MB/sec execute 494 sec latency 50.191 ms 2024-09-13T21:51:02.683 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:02 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T21:51:02.683 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:02 smithi154 ceph-mon[65435]: Upgrade: osd.6 is safe to restart 2024-09-13T21:51:02.683 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:02 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:02.684 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:02 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:51:02.684 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:02 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:02.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:02 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T21:51:02.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:02 smithi181 ceph-mon[51676]: Upgrade: osd.6 is safe to restart 2024-09-13T21:51:02.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:02 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:02.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:02 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T21:51:02.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:02 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:02.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 241521 9.91 MB/sec execute 495 sec latency 40.690 ms 2024-09-13T21:51:03.700 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:03 smithi154 ceph-mon[65435]: pgmap v240: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.5 MiB/s wr, 352 op/s; 1.4 MiB/s, 23 objects/s recovering 2024-09-13T21:51:03.701 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:03 smithi154 ceph-mon[65435]: Upgrade: Updating osd.6 2024-09-13T21:51:03.701 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:03 smithi154 ceph-mon[65435]: Deploying daemon osd.6 on smithi181 2024-09-13T21:51:03.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 241830 9.91 MB/sec execute 496 sec latency 58.851 ms 2024-09-13T21:51:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:03 smithi181 ceph-mon[51676]: pgmap v240: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.5 MiB/s wr, 352 op/s; 1.4 MiB/s, 23 objects/s recovering 2024-09-13T21:51:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:03 smithi181 ceph-mon[51676]: Upgrade: Updating osd.6 2024-09-13T21:51:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:03 smithi181 ceph-mon[51676]: Deploying daemon osd.6 on smithi181 2024-09-13T21:51:04.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 242119 9.91 MB/sec execute 497 sec latency 47.745 ms 2024-09-13T21:51:05.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:05 smithi154 ceph-mon[65435]: pgmap v241: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.3 MiB/s wr, 340 op/s; 1.2 MiB/s, 21 objects/s recovering 2024-09-13T21:51:05.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:05 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:05.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 242418 9.90 MB/sec execute 498 sec latency 47.835 ms 2024-09-13T21:51:06.034 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:05 smithi181 ceph-mon[51676]: pgmap v241: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.3 MiB/s wr, 340 op/s; 1.2 MiB/s, 21 objects/s recovering 2024-09-13T21:51:06.034 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:05 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:06.933 INFO:teuthology.orchestra.run.smithi154.stdout: 5 242713 9.89 MB/sec execute 499 sec latency 52.853 ms 2024-09-13T21:51:07.036 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:06 smithi181 systemd[1]: Stopping Ceph osd.6 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:51:07.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:07 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[35980]: 2024-09-13T21:51:07.034+0000 7f8555ae1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:51:07.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:07 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[35980]: 2024-09-13T21:51:07.034+0000 7f8555ae1700 -1 osd.6 133 *** Got signal Terminated *** 2024-09-13T21:51:07.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:07 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[35980]: 2024-09-13T21:51:07.034+0000 7f8555ae1700 -1 osd.6 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:51:07.811 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:07 smithi154 ceph-mon[65435]: pgmap v242: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 351 op/s; 825 KiB/s, 11 objects/s recovering 2024-09-13T21:51:07.811 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:07 smithi154 ceph-mon[65435]: osd.6 marked itself down and dead 2024-09-13T21:51:07.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 242963 9.88 MB/sec execute 500 sec latency 220.878 ms 2024-09-13T21:51:08.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:07 smithi181 ceph-mon[51676]: pgmap v242: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 351 op/s; 825 KiB/s, 11 objects/s recovering 2024-09-13T21:51:08.047 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:07 smithi181 ceph-mon[51676]: osd.6 marked itself down and dead 2024-09-13T21:51:08.650 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:08 smithi181 podman[60601]: 2024-09-13 21:51:08.314478267 +0000 UTC m=+1.420099621 container died 9685c71ddee817102127365cebad743e2dfa189d49578fff644c969f56c62f87 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, distribution-scope=public, vcs-type=git, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, version=8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T21:51:08.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 243050 9.86 MB/sec execute 501 sec latency 1024.573 ms 2024-09-13T21:51:08.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:08 smithi154 ceph-mon[65435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:51:08.962 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:08 smithi154 ceph-mon[65435]: osdmap e134: 8 total, 7 up, 8 in 2024-09-13T21:51:09.103 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:08 smithi181 ceph-mon[51676]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:51:09.103 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:08 smithi181 ceph-mon[51676]: osdmap e134: 8 total, 7 up, 8 in 2024-09-13T21:51:09.363 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:09 smithi181 podman[60601]: 2024-09-13 21:51:09.102660963 +0000 UTC m=+2.208282323 container cleanup 9685c71ddee817102127365cebad743e2dfa189d49578fff644c969f56c62f87 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , version=8, architecture=x86_64, io.buildah.version=1.19.8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, name=centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870) 2024-09-13T21:51:09.364 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:09 smithi181 bash[60601]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6 2024-09-13T21:51:09.650 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:09 smithi181 podman[60616]: 2024-09-13 21:51:09.362888078 +0000 UTC m=+1.044589029 container remove 9685c71ddee817102127365cebad743e2dfa189d49578fff644c969f56c62f87 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, architecture=x86_64, version=8, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, GIT_CLEAN=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-09-13T21:51:09.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:09 smithi154 ceph-mon[65435]: pgmap v244: 97 pgs: 10 stale+active+clean, 87 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 487 op/s 2024-09-13T21:51:09.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:09 smithi154 ceph-mon[65435]: osdmap e135: 8 total, 7 up, 8 in 2024-09-13T21:51:09.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 243390 9.87 MB/sec execute 502 sec latency 47.851 ms 2024-09-13T21:51:10.026 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:09 smithi181 ceph-mon[51676]: pgmap v244: 97 pgs: 10 stale+active+clean, 87 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 487 op/s 2024-09-13T21:51:10.027 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:09 smithi181 ceph-mon[51676]: osdmap e135: 8 total, 7 up, 8 in 2024-09-13T21:51:10.027 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:09 smithi181 podman[60700]: 2024-09-13 21:51:09.644376824 +0000 UTC m=+0.019455260 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:51:10.027 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:09 smithi181 podman[60700]: 2024-09-13 21:51:09.763920601 +0000 UTC m=+0.138999028 container create b6fc657ec2f4bdb87ebc05809f8f948fc565f0c15012548b0c783f2d4287c3d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-13T21:51:10.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:10 smithi181 podman[60700]: 2024-09-13 21:51:10.025693633 +0000 UTC m=+0.400772070 container init b6fc657ec2f4bdb87ebc05809f8f948fc565f0c15012548b0c783f2d4287c3d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T21:51:10.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:10 smithi181 podman[60700]: 2024-09-13 21:51:10.02975492 +0000 UTC m=+0.404833350 container start b6fc657ec2f4bdb87ebc05809f8f948fc565f0c15012548b0c783f2d4287c3d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T21:51:10.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:10 smithi181 podman[60700]: 2024-09-13 21:51:10.11731293 +0000 UTC m=+0.492391359 container attach b6fc657ec2f4bdb87ebc05809f8f948fc565f0c15012548b0c783f2d4287c3d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T21:51:10.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:10 smithi181 podman[60700]: 2024-09-13 21:51:10.268382685 +0000 UTC m=+0.643461115 container died b6fc657ec2f4bdb87ebc05809f8f948fc565f0c15012548b0c783f2d4287c3d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2) 2024-09-13T21:51:10.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 6]}]: dispatch 2024-09-13T21:51:10.769 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [3, 6]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [7, 6]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 6]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 6]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 6]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 2]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 6]}]: dispatch 2024-09-13T21:51:10.770 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:51:10.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 6]}]: dispatch 2024-09-13T21:51:10.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [3, 6]}]: dispatch 2024-09-13T21:51:10.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]: dispatch 2024-09-13T21:51:10.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:10.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [7, 6]}]: dispatch 2024-09-13T21:51:10.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]: dispatch 2024-09-13T21:51:10.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 6]}]: dispatch 2024-09-13T21:51:10.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 6]}]: dispatch 2024-09-13T21:51:10.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 6]}]: dispatch 2024-09-13T21:51:10.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 2]}]: dispatch 2024-09-13T21:51:10.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 6]}]: dispatch 2024-09-13T21:51:10.874 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:51:10.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 243775 9.87 MB/sec execute 503 sec latency 117.156 ms 2024-09-13T21:51:11.127 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:10 smithi181 podman[60723]: 2024-09-13 21:51:10.872083057 +0000 UTC m=+0.596605623 container remove b6fc657ec2f4bdb87ebc05809f8f948fc565f0c15012548b0c783f2d4287c3d7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True) 2024-09-13T21:51:11.127 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:10 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-13T21:51:11.127 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:10 smithi181 systemd[1]: Stopped Ceph osd.6 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:51:11.127 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:10 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.6.service: Consumed 4min 3.420s CPU time. 2024-09-13T21:51:11.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 systemd[1]: Starting Ceph osd.6 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:51:11.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 podman[60822]: 2024-09-13 21:51:11.239412369 +0000 UTC m=+0.015457116 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:51:11.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 podman[60822]: 2024-09-13 21:51:11.348395869 +0000 UTC m=+0.124440614 container create 46d12a7eed29761b7a1839c1aa695cb3df55328c233066129c762d21ac7027a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-13T21:51:11.762 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 podman[60822]: 2024-09-13 21:51:11.626137269 +0000 UTC m=+0.402182010 container init 46d12a7eed29761b7a1839c1aa695cb3df55328c233066129c762d21ac7027a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T21:51:11.762 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 podman[60822]: 2024-09-13 21:51:11.629542453 +0000 UTC m=+0.405587192 container start 46d12a7eed29761b7a1839c1aa695cb3df55328c233066129c762d21ac7027a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T21:51:11.763 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 podman[60822]: 2024-09-13 21:51:11.700927157 +0000 UTC m=+0.476971909 container attach 46d12a7eed29761b7a1839c1aa695cb3df55328c233066129c762d21ac7027a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: pgmap v246: 97 pgs: 4 active+undersized, 7 stale+active+clean, 10 active+undersized+degraded, 76 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.3 MiB/s wr, 351 op/s; 64/1593 objects degraded (4.018%) 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 6]}]': finished 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [3, 6]}]': finished 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]': finished 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [7, 6]}]': finished 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]': finished 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 6]}]': finished 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 6]}]': finished 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 6]}]': finished 2024-09-13T21:51:11.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 2]}]': finished 2024-09-13T21:51:11.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 6]}]': finished 2024-09-13T21:51:11.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: osdmap e136: 8 total, 7 up, 8 in 2024-09-13T21:51:11.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-mon[51676]: Health check failed: Degraded data redundancy: 64/1593 objects degraded (4.018%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:11.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 243867 9.86 MB/sec execute 504 sec latency 1117.212 ms 2024-09-13T21:51:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: pgmap v246: 97 pgs: 4 active+undersized, 7 stale+active+clean, 10 active+undersized+degraded, 76 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.3 MiB/s wr, 351 op/s; 64/1593 objects degraded (4.018%) 2024-09-13T21:51:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 6]}]': finished 2024-09-13T21:51:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [3, 6]}]': finished 2024-09-13T21:51:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]': finished 2024-09-13T21:51:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [7, 6]}]': finished 2024-09-13T21:51:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 2]}]': finished 2024-09-13T21:51:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 6]}]': finished 2024-09-13T21:51:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [4, 6]}]': finished 2024-09-13T21:51:12.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 6]}]': finished 2024-09-13T21:51:12.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 2]}]': finished 2024-09-13T21:51:12.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 6]}]': finished 2024-09-13T21:51:12.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: osdmap e136: 8 total, 7 up, 8 in 2024-09-13T21:51:12.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:11 smithi154 ceph-mon[65435]: Health check failed: Degraded data redundancy: 64/1593 objects degraded (4.018%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:12.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:51:12.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 bash[60822]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:51:12.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:51:12.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:11 smithi181 bash[60822]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:51:12.814 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-mon[51676]: osdmap e137: 8 total, 7 up, 8 in 2024-09-13T21:51:12.814 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-mon[51676]: osdmap e138: 8 total, 7 up, 8 in 2024-09-13T21:51:12.815 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T21:51:12.815 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 bash[60822]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T21:51:12.815 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T21:51:12.815 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 bash[60822]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T21:51:12.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 243915 9.84 MB/sec execute 505 sec latency 1991.769 ms 2024-09-13T21:51:12.994 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:12 smithi154 ceph-mon[65435]: osdmap e137: 8 total, 7 up, 8 in 2024-09-13T21:51:12.994 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:12 smithi154 ceph-mon[65435]: osdmap e138: 8 total, 7 up, 8 in 2024-09-13T21:51:13.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T21:51:13.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 bash[60822]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T21:51:13.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T21:51:13.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 bash[60822]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T21:51:13.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T21:51:13.151 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 bash[60822]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T21:51:13.151 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T21:51:13.151 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 bash[60822]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T21:51:13.151 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate[60837]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T21:51:13.151 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 bash[60822]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T21:51:13.151 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:12 smithi181 podman[60822]: 2024-09-13 21:51:12.854917341 +0000 UTC m=+1.630962094 container died 46d12a7eed29761b7a1839c1aa695cb3df55328c233066129c762d21ac7027a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T21:51:13.733 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:13 smithi181 podman[60969]: 2024-09-13 21:51:13.481077183 +0000 UTC m=+0.618217234 container remove 46d12a7eed29761b7a1839c1aa695cb3df55328c233066129c762d21ac7027a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T21:51:13.733 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:13 smithi181 podman[61024]: 2024-09-13 21:51:13.606948528 +0000 UTC m=+0.019537899 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:51:13.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:13 smithi154 ceph-mon[65435]: pgmap v249: 97 pgs: 1 remapped+peering, 2 peering, 10 active+undersized, 1 stale+active+clean, 23 active+undersized+degraded, 60 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s wr, 221 op/s; 144/1587 objects degraded (9.074%) 2024-09-13T21:51:13.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 244313 9.85 MB/sec execute 506 sec latency 68.762 ms 2024-09-13T21:51:14.056 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:13 smithi181 podman[61024]: 2024-09-13 21:51:13.732291715 +0000 UTC m=+0.144881064 container create 07e99a632a762b8c01f5a11a40fa383f7717431e5055380be810e17bcf26728a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T21:51:14.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:13 smithi181 ceph-mon[51676]: pgmap v249: 97 pgs: 1 remapped+peering, 2 peering, 10 active+undersized, 1 stale+active+clean, 23 active+undersized+degraded, 60 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 MiB/s wr, 221 op/s; 144/1587 objects degraded (9.074%) 2024-09-13T21:51:14.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:14 smithi181 podman[61024]: 2024-09-13 21:51:14.055017847 +0000 UTC m=+0.467607206 container init 07e99a632a762b8c01f5a11a40fa383f7717431e5055380be810e17bcf26728a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-13T21:51:14.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:14 smithi181 podman[61024]: 2024-09-13 21:51:14.059475317 +0000 UTC m=+0.472064674 container start 07e99a632a762b8c01f5a11a40fa383f7717431e5055380be810e17bcf26728a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-09-13T21:51:14.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:14 smithi181 ceph-osd[61042]: -- 172.21.15.181:0/3326338284 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c4faedc1e0 con 0x55c4faeb0000 2024-09-13T21:51:14.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:14 smithi181 bash[61024]: 07e99a632a762b8c01f5a11a40fa383f7717431e5055380be810e17bcf26728a 2024-09-13T21:51:14.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:14 smithi181 systemd[1]: Started Ceph osd.6 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:51:14.930 INFO:teuthology.orchestra.run.smithi154.stdout: 5 244552 9.84 MB/sec execute 507 sec latency 38.867 ms 2024-09-13T21:51:15.551 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:15.551 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:15.551 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:15.552 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:15 smithi154 ceph-mon[65435]: pgmap v251: 97 pgs: 1 active+clean+remapped, 2 remapped+peering, 2 peering, 10 active+undersized, 1 stale+active+clean, 23 active+undersized+degraded, 58 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.0 MiB/s wr, 289 op/s; 147/1596 objects degraded (9.211%); 9/1596 objects misplaced (0.564%) 2024-09-13T21:51:15.552 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:15 smithi154 ceph-mon[65435]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T21:51:15.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:15.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:15.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:15.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:15 smithi181 ceph-mon[51676]: pgmap v251: 97 pgs: 1 active+clean+remapped, 2 remapped+peering, 2 peering, 10 active+undersized, 1 stale+active+clean, 23 active+undersized+degraded, 58 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.0 MiB/s wr, 289 op/s; 147/1596 objects degraded (9.211%); 9/1596 objects misplaced (0.564%) 2024-09-13T21:51:15.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:15 smithi181 ceph-mon[51676]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T21:51:15.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 244789 9.83 MB/sec execute 508 sec latency 878.107 ms 2024-09-13T21:51:16.336 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:15 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:51:15.934+0000 7f4fde7bc640 -1 Falling back to public interface 2024-09-13T21:51:16.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:16 smithi154 ceph-mon[65435]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:51:16.616 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:16 smithi154 ceph-mon[65435]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T21:51:16.655 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:16 smithi181 ceph-mon[51676]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:51:16.655 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:16 smithi181 ceph-mon[51676]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T21:51:16.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 245061 9.83 MB/sec execute 509 sec latency 45.840 ms 2024-09-13T21:51:17.661 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:51:17.905 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:17 smithi154 ceph-mon[65435]: pgmap v254: 97 pgs: 6 active+clean+remapped, 2 remapped+peering, 10 active+undersized, 1 stale+active+clean, 23 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 357 op/s; 155/1659 objects degraded (9.343%); 57/1659 objects misplaced (3.436%) 2024-09-13T21:51:17.905 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:17 smithi154 ceph-mon[65435]: osdmap e141: 8 total, 7 up, 8 in 2024-09-13T21:51:17.905 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:17 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 155/1659 objects degraded (9.343%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:17.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 245281 9.82 MB/sec execute 510 sec latency 526.370 ms 2024-09-13T21:51:18.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:17 smithi181 ceph-mon[51676]: pgmap v254: 97 pgs: 6 active+clean+remapped, 2 remapped+peering, 10 active+undersized, 1 stale+active+clean, 23 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 357 op/s; 155/1659 objects degraded (9.343%); 57/1659 objects misplaced (3.436%) 2024-09-13T21:51:18.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:17 smithi181 ceph-mon[51676]: osdmap e141: 8 total, 7 up, 8 in 2024-09-13T21:51:18.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:17 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 155/1659 objects degraded (9.343%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:18.505 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:51:18.505 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (8m) 111s ago 16m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:51:18.505 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (5m) 111s ago 16m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (4m) 0s ago 16m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (8m) 111s ago 16m 52.7M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (11m) 111s ago 11m 37.3M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (11m) 0s ago 11m 17.8M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (7m) 111s ago 18m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (6m) 0s ago 16m 443M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (6m) 111s ago 18m 50.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (5m) 0s ago 16m 43.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (10m) 111s ago 10m 279M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (10m) 0s ago 10m 84.8M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (8m) 111s ago 16m 15.4M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (7m) 0s ago 16m 15.2M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:51:18.506 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (4m) 111s ago 15m 333M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:51:18.507 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (3m) 111s ago 15m 257M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:51:18.507 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (2m) 111s ago 14m 201M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:51:18.507 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (2m) 111s ago 14m 14.2M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:51:18.507 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (77s) 0s ago 13m 300M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:51:18.507 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (41s) 0s ago 13m 233M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 80819532f2ce 2024-09-13T21:51:18.507 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (4s) 0s ago 12m 11.2M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 07e99a632a76 2024-09-13T21:51:18.507 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (12m) 0s ago 12m 796M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:51:18.507 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (6m) 111s ago 15m 66.1M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:51:18.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 245534 9.82 MB/sec execute 511 sec latency 61.478 ms 2024-09-13T21:51:18.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:18 smithi154 ceph-mon[65435]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:18.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:18 smithi154 ceph-mon[65435]: osdmap e142: 8 total, 7 up, 8 in 2024-09-13T21:51:18.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:18 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:18.932 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:18 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:18.989 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:51:18.989 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:51:18.989 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:51:18.990 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T21:51:18.991 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 10 2024-09-13T21:51:18.991 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:51:18.991 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:51:19.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:18 smithi181 ceph-mon[51676]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:19.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:18 smithi181 ceph-mon[51676]: osdmap e142: 8 total, 7 up, 8 in 2024-09-13T21:51:19.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:18 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:19.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:18 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:19.505 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "13/23 daemons upgraded", 2024-09-13T21:51:19.506 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:51:19.507 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:51:19.507 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:51:19.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 245833 9.81 MB/sec execute 512 sec latency 61.808 ms 2024-09-13T21:51:19.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:19 smithi154 ceph-mon[65435]: from='client.44337 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:19.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:19 smithi154 ceph-mon[65435]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:19.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:19 smithi154 ceph-mon[65435]: pgmap v257: 97 pgs: 1 creating+remapped+peering, 8 active+clean+remapped, 10 active+undersized, 23 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.1 MiB/s wr, 370 op/s; 153/1647 objects degraded (9.290%); 68/1647 objects misplaced (4.129%); 5.1 KiB/s, 299 objects/s recovering 2024-09-13T21:51:19.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:19 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/135051252' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:19.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:19.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:19.975 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:19 smithi154 ceph-mon[65435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 153/1647 objects degraded (9.290%), 23 pgs degraded 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: osd.6 (root=default,host=smithi181) is down 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 153/1647 objects degraded (9.290%), 23 pgs degraded 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-13T21:51:20.074 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-13T21:51:20.075 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-13T21:51:20.076 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-13T21:51:20.120 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:19 smithi181 ceph-mon[51676]: from='client.44337 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:20.120 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:19 smithi181 ceph-mon[51676]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:20.120 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:19 smithi181 ceph-mon[51676]: pgmap v257: 97 pgs: 1 creating+remapped+peering, 8 active+clean+remapped, 10 active+undersized, 23 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.1 MiB/s wr, 370 op/s; 153/1647 objects degraded (9.290%); 68/1647 objects misplaced (4.129%); 5.1 KiB/s, 299 objects/s recovering 2024-09-13T21:51:20.120 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:19 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/135051252' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:20.120 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:20.120 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:20.121 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:19 smithi181 ceph-mon[51676]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T21:51:20.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 246270 9.81 MB/sec execute 513 sec latency 39.138 ms 2024-09-13T21:51:20.993 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:20 smithi154 ceph-mon[65435]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:20.993 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:20 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/4287628411' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:51:21.123 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:20 smithi181 ceph-mon[51676]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:21.123 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:20 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/4287628411' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:51:21.812 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:21 smithi154 ceph-mon[65435]: pgmap v258: 97 pgs: 1 creating+remapped+peering, 8 active+clean+remapped, 10 active+undersized, 23 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 359 op/s; 154/1662 objects degraded (9.266%); 69/1662 objects misplaced (4.152%); 3.5 KiB/s, 206 objects/s recovering 2024-09-13T21:51:21.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 246576 9.80 MB/sec execute 514 sec latency 39.054 ms 2024-09-13T21:51:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:21 smithi181 ceph-mon[51676]: pgmap v258: 97 pgs: 1 creating+remapped+peering, 8 active+clean+remapped, 10 active+undersized, 23 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 359 op/s; 154/1662 objects degraded (9.266%); 69/1662 objects misplaced (4.152%); 3.5 KiB/s, 206 objects/s recovering 2024-09-13T21:51:22.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 246855 9.80 MB/sec execute 515 sec latency 112.504 ms 2024-09-13T21:51:23.076 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:22 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 157/1614 objects degraded (9.727%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:23.116 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:22 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 157/1614 objects degraded (9.727%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:23.847 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:51:23.535+0000 7f4fde7bc640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-13T21:51:23.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 247167 9.81 MB/sec execute 516 sec latency 69.702 ms 2024-09-13T21:51:24.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: pgmap v259: 97 pgs: 1 creating+remapped+peering, 8 active+clean+remapped, 10 active+undersized, 23 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 157/1614 objects degraded (9.727%); 65/1614 objects misplaced (4.027%); 3.0 KiB/s, 176 objects/s recovering 2024-09-13T21:51:24.097 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:24.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:24.099 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: pgmap v259: 97 pgs: 1 creating+remapped+peering, 8 active+clean+remapped, 10 active+undersized, 23 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 157/1614 objects degraded (9.727%); 65/1614 objects misplaced (4.027%); 3.0 KiB/s, 176 objects/s recovering 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:24.852 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:51:24.531+0000 7f4fde7bc640 -1 osd.6 133 log_to_monitors true 2024-09-13T21:51:24.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 247481 9.80 MB/sec execute 517 sec latency 55.011 ms 2024-09-13T21:51:25.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:24 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:25.101 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:24 smithi154 ceph-mon[65435]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T21:51:25.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:24 smithi154 ceph-mon[65435]: from='osd.6 [v2:172.21.15.181:6816/3842004260,v1:172.21.15.181:6817/3842004260]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:51:25.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:24 smithi154 ceph-mon[65435]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:51:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:24 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:24 smithi181 ceph-mon[51676]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T21:51:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:24 smithi181 ceph-mon[51676]: from='osd.6 [v2:172.21.15.181:6816/3842004260,v1:172.21.15.181:6817/3842004260]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:51:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:24 smithi181 ceph-mon[51676]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T21:51:25.650 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:51:25.328+0000 7f4fd255f640 -1 osd.6 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:51:25.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:25 smithi154 ceph-mon[65435]: pgmap v260: 97 pgs: 9 active+clean+remapped, 10 active+undersized, 23 active+undersized+degraded, 55 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.0 MiB/s wr, 346 op/s; 161/1641 objects degraded (9.811%); 76/1641 objects misplaced (4.631%); 2.6 KiB/s, 149 objects/s recovering 2024-09-13T21:51:25.889 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:25 smithi154 ceph-mon[65435]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T21:51:25.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:25 smithi154 ceph-mon[65435]: osdmap e143: 8 total, 7 up, 8 in 2024-09-13T21:51:25.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:25 smithi154 ceph-mon[65435]: from='osd.6 [v2:172.21.15.181:6816/3842004260,v1:172.21.15.181:6817/3842004260]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:51:25.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:25 smithi154 ceph-mon[65435]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:51:25.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:25 smithi154 ceph-mon[65435]: from='osd.6 ' entity='osd.6' 2024-09-13T21:51:25.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:25.890 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:51:25.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 247739 9.80 MB/sec execute 518 sec latency 84.261 ms 2024-09-13T21:51:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-mon[51676]: pgmap v260: 97 pgs: 9 active+clean+remapped, 10 active+undersized, 23 active+undersized+degraded, 55 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.0 MiB/s wr, 346 op/s; 161/1641 objects degraded (9.811%); 76/1641 objects misplaced (4.631%); 2.6 KiB/s, 149 objects/s recovering 2024-09-13T21:51:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-mon[51676]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T21:51:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-mon[51676]: osdmap e143: 8 total, 7 up, 8 in 2024-09-13T21:51:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-mon[51676]: from='osd.6 [v2:172.21.15.181:6816/3842004260,v1:172.21.15.181:6817/3842004260]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:51:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-mon[51676]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:51:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-mon[51676]: from='osd.6 ' entity='osd.6' 2024-09-13T21:51:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:51:26.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 247944 9.79 MB/sec execute 519 sec latency 589.213 ms 2024-09-13T21:51:26.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:26 smithi154 ceph-mon[65435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:51:26.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:26 smithi154 ceph-mon[65435]: osd.6 [v2:172.21.15.181:6816/3842004260,v1:172.21.15.181:6817/3842004260] boot 2024-09-13T21:51:26.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:26 smithi154 ceph-mon[65435]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T21:51:26.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:26 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:51:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:26 smithi181 ceph-mon[51676]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:51:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:26 smithi181 ceph-mon[51676]: osd.6 [v2:172.21.15.181:6816/3842004260,v1:172.21.15.181:6817/3842004260] boot 2024-09-13T21:51:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:26 smithi181 ceph-mon[51676]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T21:51:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:26 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T21:51:27.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 248071 9.77 MB/sec execute 520 sec latency 1556.517 ms 2024-09-13T21:51:28.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:27 smithi181 ceph-mon[51676]: pgmap v263: 97 pgs: 2 remapped+peering, 7 peering, 7 active+clean+remapped, 9 active+undersized, 17 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.7 MiB/s wr, 405 op/s; 112/1650 objects degraded (6.788%); 59/1650 objects misplaced (3.576%) 2024-09-13T21:51:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:27 smithi181 ceph-mon[51676]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T21:51:28.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:27 smithi154 ceph-mon[65435]: pgmap v263: 97 pgs: 2 remapped+peering, 7 peering, 7 active+clean+remapped, 9 active+undersized, 17 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.7 MiB/s wr, 405 op/s; 112/1650 objects degraded (6.788%); 59/1650 objects misplaced (3.576%) 2024-09-13T21:51:28.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:27 smithi154 ceph-mon[65435]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T21:51:28.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 248148 9.76 MB/sec execute 521 sec latency 2281.176 ms 2024-09-13T21:51:29.551 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:29 smithi154 ceph-mon[65435]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T21:51:29.551 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:29 smithi154 ceph-mon[65435]: pgmap v266: 97 pgs: 2 remapped+peering, 14 peering, 6 active+clean+remapped, 4 active+undersized, 16 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 6.1 MiB/s wr, 628 op/s; 107/1671 objects degraded (6.403%); 45/1671 objects misplaced (2.693%) 2024-09-13T21:51:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:29 smithi181 ceph-mon[51676]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T21:51:29.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:29 smithi181 ceph-mon[51676]: pgmap v266: 97 pgs: 2 remapped+peering, 14 peering, 6 active+clean+remapped, 4 active+undersized, 16 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 6.1 MiB/s wr, 628 op/s; 107/1671 objects degraded (6.403%); 45/1671 objects misplaced (2.693%) 2024-09-13T21:51:29.931 INFO:teuthology.orchestra.run.smithi154.stdout: 5 248372 9.75 MB/sec execute 522 sec latency 81.036 ms 2024-09-13T21:51:30.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:30 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 107/1671 objects degraded (6.403%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:30.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:30 smithi154 ceph-mon[65435]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T21:51:30.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:30 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 107/1671 objects degraded (6.403%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:30.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:30 smithi181 ceph-mon[51676]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T21:51:30.940 INFO:teuthology.orchestra.run.smithi154.stdout: 5 248556 9.74 MB/sec execute 523 sec latency 99.874 ms 2024-09-13T21:51:31.351 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:31 smithi154 ceph-mon[65435]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T21:51:31.351 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:31 smithi154 ceph-mon[65435]: pgmap v269: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 16 peering, 3 active+clean+remapped, 73 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.3 MiB/s wr, 359 op/s; 667/1620 objects degraded (41.173%); 27/1620 objects misplaced (1.667%); 1.4 MiB/s, 24 objects/s recovering 2024-09-13T21:51:31.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:31 smithi181 ceph-mon[51676]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T21:51:31.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:31 smithi181 ceph-mon[51676]: pgmap v269: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 16 peering, 3 active+clean+remapped, 73 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.3 MiB/s wr, 359 op/s; 667/1620 objects degraded (41.173%); 27/1620 objects misplaced (1.667%); 1.4 MiB/s, 24 objects/s recovering 2024-09-13T21:51:31.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 248811 9.74 MB/sec execute 524 sec latency 105.173 ms 2024-09-13T21:51:32.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 249067 9.74 MB/sec execute 525 sec latency 89.982 ms 2024-09-13T21:51:33.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 249416 9.73 MB/sec execute 526 sec latency 238.641 ms 2024-09-13T21:51:33.946 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:33 smithi154 ceph-mon[65435]: pgmap v270: 97 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 87 active+clean; 344 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 498 op/s; 2543/1545 objects degraded (164.595%); 5/1545 objects misplaced (0.324%); 1.1 MiB/s, 46 keys/s, 80 objects/s recovering 2024-09-13T21:51:33.946 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:33 smithi154 ceph-mon[65435]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T21:51:34.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:33 smithi181 ceph-mon[51676]: pgmap v270: 97 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 87 active+clean; 344 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 498 op/s; 2543/1545 objects degraded (164.595%); 5/1545 objects misplaced (0.324%); 1.1 MiB/s, 46 keys/s, 80 objects/s recovering 2024-09-13T21:51:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:33 smithi181 ceph-mon[51676]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T21:51:34.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 249694 9.72 MB/sec execute 527 sec latency 1027.711 ms 2024-09-13T21:51:34.954 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:34 smithi154 ceph-mon[65435]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T21:51:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:34 smithi181 ceph-mon[51676]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T21:51:35.724 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:35 smithi154 ceph-mon[65435]: pgmap v273: 97 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 88 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.7 MiB/s wr, 274 op/s; 2543/1548 objects degraded (164.276%); 5/1548 objects misplaced (0.323%); 970 KiB/s, 46 keys/s, 71 objects/s recovering 2024-09-13T21:51:35.724 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:35 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 2543/1548 objects degraded (164.276%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:35.724 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:35 smithi154 ceph-mon[65435]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T21:51:35.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 249863 9.71 MB/sec execute 528 sec latency 75.666 ms 2024-09-13T21:51:36.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:35 smithi181 ceph-mon[51676]: pgmap v273: 97 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 88 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.7 MiB/s wr, 274 op/s; 2543/1548 objects degraded (164.276%); 5/1548 objects misplaced (0.323%); 970 KiB/s, 46 keys/s, 71 objects/s recovering 2024-09-13T21:51:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:35 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 2543/1548 objects degraded (164.276%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:35 smithi181 ceph-mon[51676]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T21:51:36.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 250141 9.72 MB/sec execute 529 sec latency 70.390 ms 2024-09-13T21:51:37.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:37 smithi154 ceph-mon[65435]: pgmap v275: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.3 MiB/s wr, 421 op/s; 1876/1611 objects degraded (116.449%); 842 KiB/s, 40 keys/s, 232 objects/s recovering 2024-09-13T21:51:37.782 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:37 smithi154 ceph-mon[65435]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T21:51:37.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 250502 9.71 MB/sec execute 530 sec latency 71.045 ms 2024-09-13T21:51:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:37 smithi181 ceph-mon[51676]: pgmap v275: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.3 MiB/s wr, 421 op/s; 1876/1611 objects degraded (116.449%); 842 KiB/s, 40 keys/s, 232 objects/s recovering 2024-09-13T21:51:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:37 smithi181 ceph-mon[51676]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T21:51:38.807 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:38 smithi154 ceph-mon[65435]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T21:51:38.808 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:38 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:38.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 250822 9.71 MB/sec execute 531 sec latency 62.151 ms 2024-09-13T21:51:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:38 smithi181 ceph-mon[51676]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T21:51:39.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:38 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:39.828 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:39 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:39.829 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:39 smithi154 ceph-mon[65435]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T21:51:39.829 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:39 smithi154 ceph-mon[65435]: pgmap v278: 97 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 MiB/s wr, 277 op/s; 1876/1608 objects degraded (116.667%); 969 KiB/s, 219 objects/s recovering 2024-09-13T21:51:39.829 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:39 smithi154 ceph-mon[65435]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T21:51:39.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 251110 9.71 MB/sec execute 532 sec latency 45.173 ms 2024-09-13T21:51:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:39 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:39 smithi181 ceph-mon[51676]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T21:51:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:39 smithi181 ceph-mon[51676]: pgmap v278: 97 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 MiB/s wr, 277 op/s; 1876/1608 objects degraded (116.667%); 969 KiB/s, 219 objects/s recovering 2024-09-13T21:51:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:39 smithi181 ceph-mon[51676]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T21:51:40.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 251370 9.70 MB/sec execute 533 sec latency 59.518 ms 2024-09-13T21:51:41.605 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:41.605 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:51:41.605 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:41 smithi154 ceph-mon[65435]: pgmap v280: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 3.4 MiB/s wr, 332 op/s; 1230/1605 objects degraded (76.636%); 807 KiB/s, 293 objects/s recovering 2024-09-13T21:51:41.605 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:41 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 1876/1608 objects degraded (116.667%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:51:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:41 smithi181 ceph-mon[51676]: pgmap v280: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 3.4 MiB/s wr, 332 op/s; 1230/1605 objects degraded (76.636%); 807 KiB/s, 293 objects/s recovering 2024-09-13T21:51:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:41 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 1876/1608 objects degraded (116.667%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:41.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 251714 9.70 MB/sec execute 534 sec latency 55.960 ms 2024-09-13T21:51:42.621 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:42 smithi154 ceph-mon[65435]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T21:51:42.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:42 smithi181 ceph-mon[51676]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T21:51:42.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 251918 9.69 MB/sec execute 535 sec latency 74.507 ms 2024-09-13T21:51:43.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:43 smithi154 ceph-mon[65435]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T21:51:43.666 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:43 smithi154 ceph-mon[65435]: pgmap v283: 97 pgs: 2 active+recovering+undersized+degraded+remapped, 95 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 578 op/s; 17/1629 objects degraded (1.044%); 7/1629 objects misplaced (0.430%); 0 B/s, 255 objects/s recovering 2024-09-13T21:51:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:43 smithi181 ceph-mon[51676]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T21:51:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:43 smithi181 ceph-mon[51676]: pgmap v283: 97 pgs: 2 active+recovering+undersized+degraded+remapped, 95 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 578 op/s; 17/1629 objects degraded (1.044%); 7/1629 objects misplaced (0.430%); 0 B/s, 255 objects/s recovering 2024-09-13T21:51:43.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 252104 9.68 MB/sec execute 536 sec latency 62.360 ms 2024-09-13T21:51:44.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 252358 9.69 MB/sec execute 537 sec latency 76.587 ms 2024-09-13T21:51:45.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:45 smithi154 ceph-mon[65435]: pgmap v284: 97 pgs: 2 active+recovering+undersized+degraded+remapped, 95 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 479 op/s; 17/1617 objects degraded (1.051%); 7/1617 objects misplaced (0.433%); 0 B/s, 205 objects/s recovering 2024-09-13T21:51:45.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 252650 9.68 MB/sec execute 538 sec latency 89.132 ms 2024-09-13T21:51:46.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:45 smithi181 ceph-mon[51676]: pgmap v284: 97 pgs: 2 active+recovering+undersized+degraded+remapped, 95 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 479 op/s; 17/1617 objects degraded (1.051%); 7/1617 objects misplaced (0.433%); 0 B/s, 205 objects/s recovering 2024-09-13T21:51:46.796 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:46 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 17/1617 objects degraded (1.051%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:46.796 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:46 smithi154 ceph-mon[65435]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T21:51:46.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 252938 9.68 MB/sec execute 539 sec latency 840.996 ms 2024-09-13T21:51:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:46 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 17/1617 objects degraded (1.051%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T21:51:47.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:46 smithi181 ceph-mon[51676]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T21:51:47.819 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:47 smithi154 ceph-mon[65435]: pgmap v286: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.2 MiB/s wr, 450 op/s; 6/1608 objects degraded (0.373%); 7/1608 objects misplaced (0.435%); 131 KiB/s, 114 objects/s recovering 2024-09-13T21:51:47.820 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:47 smithi154 ceph-mon[65435]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T21:51:47.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 253217 9.67 MB/sec execute 540 sec latency 86.301 ms 2024-09-13T21:51:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:47 smithi181 ceph-mon[51676]: pgmap v286: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.2 MiB/s wr, 450 op/s; 6/1608 objects degraded (0.373%); 7/1608 objects misplaced (0.435%); 131 KiB/s, 114 objects/s recovering 2024-09-13T21:51:48.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:47 smithi181 ceph-mon[51676]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T21:51:48.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 253495 9.66 MB/sec execute 541 sec latency 57.662 ms 2024-09-13T21:51:49.877 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:49 smithi154 ceph-mon[65435]: pgmap v288: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.0 MiB/s wr, 283 op/s; 6/1596 objects degraded (0.376%); 7/1596 objects misplaced (0.439%); 129 KiB/s, 17 objects/s recovering 2024-09-13T21:51:49.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 253850 9.66 MB/sec execute 542 sec latency 54.090 ms 2024-09-13T21:51:50.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:49 smithi181 ceph-mon[51676]: pgmap v288: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.0 MiB/s wr, 283 op/s; 6/1596 objects degraded (0.376%); 7/1596 objects misplaced (0.439%); 129 KiB/s, 17 objects/s recovering 2024-09-13T21:51:50.718 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:51:50.903 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:50 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 6/1596 objects degraded (0.376%), 1 pg degraded (PG_DEGRADED) 2024-09-13T21:51:50.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 254221 9.66 MB/sec execute 543 sec latency 49.480 ms 2024-09-13T21:51:51.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:50 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 6/1596 objects degraded (0.376%), 1 pg degraded (PG_DEGRADED) 2024-09-13T21:51:51.843 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (8m) 2m ago 17m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (5m) 2m ago 17m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (5m) 33s ago 16m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (8m) 2m ago 17m 52.7M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (11m) 2m ago 11m 37.3M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (11m) 33s ago 11m 17.8M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (8m) 2m ago 18m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (7m) 33s ago 16m 443M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (6m) 2m ago 18m 50.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (6m) 33s ago 16m 43.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (11m) 2m ago 11m 279M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:51:51.844 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (11m) 33s ago 11m 84.8M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (8m) 2m ago 17m 15.4M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (8m) 33s ago 16m 15.2M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (4m) 2m ago 16m 333M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (4m) 2m ago 15m 257M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (3m) 2m ago 15m 201M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (2m) 2m ago 14m 14.2M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (110s) 33s ago 14m 300M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (75s) 33s ago 13m 233M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 80819532f2ce 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (38s) 33s ago 13m 11.2M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 07e99a632a76 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (13m) 33s ago 13m 796M 2703M 17.2.0 e1d6a67b021e eda179203ece 2024-09-13T21:51:51.845 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (7m) 2m ago 16m 66.1M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:51:51.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:51 smithi154 ceph-mon[65435]: pgmap v289: 97 pgs: 97 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.2 MiB/s wr, 324 op/s; 164 KiB/s, 80 objects/s recovering 2024-09-13T21:51:51.930 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:51 smithi154 ceph-mon[65435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/1596 objects degraded (0.376%), 1 pg degraded) 2024-09-13T21:51:51.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:51 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:51:51.931 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:51 smithi154 ceph-mon[65435]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:51.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 254549 9.66 MB/sec execute 544 sec latency 55.888 ms 2024-09-13T21:51:52.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:51 smithi181 ceph-mon[51676]: pgmap v289: 97 pgs: 97 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.2 MiB/s wr, 324 op/s; 164 KiB/s, 80 objects/s recovering 2024-09-13T21:51:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:51 smithi181 ceph-mon[51676]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/1596 objects degraded (0.376%), 1 pg degraded) 2024-09-13T21:51:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:51 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:51:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:51 smithi181 ceph-mon[51676]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:52.502 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 7 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:51:52.503 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:51:52.504 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:51:52.504 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:51:52.504 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T21:51:52.504 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 11 2024-09-13T21:51:52.504 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:51:52.504 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:51:52.696 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:52 smithi154 ceph-mon[65435]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:52.697 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:52 smithi154 ceph-mon[65435]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:52.697 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:52 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/3511600163' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:52.935 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "13/23 daemons upgraded", 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T21:51:52.936 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:51:52.937 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:51:52.946 INFO:teuthology.orchestra.run.smithi154.stdout: 5 254836 9.66 MB/sec execute 545 sec latency 82.339 ms 2024-09-13T21:51:53.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:52 smithi181 ceph-mon[51676]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:53.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:52 smithi181 ceph-mon[51676]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:53.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:52 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/3511600163' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:51:53.494 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:51:53.715 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:53 smithi154 ceph-mon[65435]: pgmap v290: 97 pgs: 97 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.7 MiB/s wr, 452 op/s; 164 KiB/s, 80 objects/s recovering 2024-09-13T21:51:53.715 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:53 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:53.715 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:53 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2269584630' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:51:53.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 255163 9.66 MB/sec execute 546 sec latency 58.205 ms 2024-09-13T21:51:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:53 smithi181 ceph-mon[51676]: pgmap v290: 97 pgs: 97 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.7 MiB/s wr, 452 op/s; 164 KiB/s, 80 objects/s recovering 2024-09-13T21:51:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:53 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:53 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2269584630' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:51:54.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:54 smithi154 ceph-mon[65435]: from='client.44373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:54.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:54 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:54.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:54 smithi154 ceph-mon[65435]: Upgrade: osd.7 is safe to restart 2024-09-13T21:51:54.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 255381 9.65 MB/sec execute 547 sec latency 73.319 ms 2024-09-13T21:51:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:54 smithi181 ceph-mon[51676]: from='client.44373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:51:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:54 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T21:51:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:54 smithi181 ceph-mon[51676]: Upgrade: osd.7 is safe to restart 2024-09-13T21:51:55.791 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:55 smithi154 ceph-mon[65435]: pgmap v291: 97 pgs: 97 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.8 MiB/s wr, 383 op/s; 59 KiB/s, 60 objects/s recovering 2024-09-13T21:51:55.791 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:55.791 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:51:55.792 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:55.792 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:55.792 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:51:55.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 255555 9.64 MB/sec execute 548 sec latency 89.298 ms 2024-09-13T21:51:55.983 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:55 smithi181 ceph-mon[51676]: pgmap v291: 97 pgs: 97 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.8 MiB/s wr, 383 op/s; 59 KiB/s, 60 objects/s recovering 2024-09-13T21:51:55.983 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:55.983 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T21:51:55.983 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:51:55.983 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:51:55.983 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:51:56.831 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:56 smithi154 ceph-mon[65435]: Upgrade: Updating osd.7 2024-09-13T21:51:56.831 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:56 smithi154 ceph-mon[65435]: Deploying daemon osd.7 on smithi181 2024-09-13T21:51:56.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 255808 9.64 MB/sec execute 549 sec latency 82.124 ms 2024-09-13T21:51:57.082 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:56 smithi181 ceph-mon[51676]: Upgrade: Updating osd.7 2024-09-13T21:51:57.083 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:56 smithi181 ceph-mon[51676]: Deploying daemon osd.7 on smithi181 2024-09-13T21:51:57.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:57 smithi181 ceph-mon[51676]: pgmap v292: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 431 op/s; 53 KiB/s, 53 objects/s recovering 2024-09-13T21:51:57.904 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:57 smithi154 ceph-mon[65435]: pgmap v292: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 431 op/s; 53 KiB/s, 53 objects/s recovering 2024-09-13T21:51:57.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 256017 9.64 MB/sec execute 550 sec latency 81.044 ms 2024-09-13T21:51:58.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:51:58 smithi181 systemd[1]: Stopping Ceph osd.7 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:51:58.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 256254 9.62 MB/sec execute 551 sec latency 87.776 ms 2024-09-13T21:51:59.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:51:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[39058]: 2024-09-13T21:51:59.215+0000 7f4772dc7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:51:59.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:51:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[39058]: 2024-09-13T21:51:59.215+0000 7f4772dc7700 -1 osd.7 158 *** Got signal Terminated *** 2024-09-13T21:51:59.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:51:59 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[39058]: 2024-09-13T21:51:59.215+0000 7f4772dc7700 -1 osd.7 158 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:51:59.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 256459 9.62 MB/sec execute 552 sec latency 231.037 ms 2024-09-13T21:52:00.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:59 smithi154 ceph-mon[65435]: pgmap v293: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 448 op/s; 44 KiB/s, 45 objects/s recovering 2024-09-13T21:52:00.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:51:59 smithi154 ceph-mon[65435]: osd.7 marked itself down and dead 2024-09-13T21:52:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:59 smithi181 ceph-mon[51676]: pgmap v293: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 448 op/s; 44 KiB/s, 45 objects/s recovering 2024-09-13T21:52:00.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:51:59 smithi181 ceph-mon[51676]: osd.7 marked itself down and dead 2024-09-13T21:52:00.716 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:00 smithi181 podman[62853]: 2024-09-13 21:52:00.412764836 +0000 UTC m=+1.352611989 container died eda179203ece2afdd833a518ea082e59a9131c4f34f829c298ad388008aa4a7b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, RELEASE=HEAD, name=centos-stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870) 2024-09-13T21:52:00.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 256495 9.60 MB/sec execute 553 sec latency 1070.140 ms 2024-09-13T21:52:00.977 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:00 smithi154 ceph-mon[65435]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:52:00.977 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:00 smithi154 ceph-mon[65435]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T21:52:00.977 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:00 smithi154 ceph-mon[65435]: osdmap e159: 8 total, 7 up, 8 in 2024-09-13T21:52:00.977 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:00 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:01.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:00 smithi181 ceph-mon[51676]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T21:52:01.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:00 smithi181 ceph-mon[51676]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T21:52:01.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:00 smithi181 ceph-mon[51676]: osdmap e159: 8 total, 7 up, 8 in 2024-09-13T21:52:01.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:00 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:01.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:00 smithi181 podman[62853]: 2024-09-13 21:52:00.822938006 +0000 UTC m=+1.762785145 container cleanup eda179203ece2afdd833a518ea082e59a9131c4f34f829c298ad388008aa4a7b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, com.redhat.component=centos-stream-container, name=centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_BRANCH=HEAD, release=754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, distribution-scope=public, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T21:52:01.151 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:00 smithi181 bash[62853]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7 2024-09-13T21:52:01.541 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:01 smithi181 podman[62874]: 2024-09-13 21:52:01.166215915 +0000 UTC m=+0.747286569 container remove eda179203ece2afdd833a518ea082e59a9131c4f34f829c298ad388008aa4a7b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , version=8, io.k8s.display-name=CentOS Stream 8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_CLEAN=True, release=754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, name=centos-stream, architecture=x86_64, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-13T21:52:01.804 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:01 smithi181 podman[62959]: 2024-09-13 21:52:01.443623499 +0000 UTC m=+0.017094853 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:52:01.804 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:01 smithi181 podman[62959]: 2024-09-13 21:52:01.567435054 +0000 UTC m=+0.140906450 container create ee73f9df3842cca5f1b5e2857f02c344f8819d0749108d416e31adfae20d44bb (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be) 2024-09-13T21:52:01.805 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:01 smithi181 ceph-mon[51676]: pgmap v295: 97 pgs: 13 peering, 10 stale+active+clean, 74 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-13T21:52:01.805 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:01 smithi181 ceph-mon[51676]: osdmap e160: 8 total, 7 up, 8 in 2024-09-13T21:52:01.805 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:01 smithi181 ceph-mon[51676]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:52:01.886 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:01 smithi154 ceph-mon[65435]: pgmap v295: 97 pgs: 13 peering, 10 stale+active+clean, 74 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-13T21:52:01.886 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:01 smithi154 ceph-mon[65435]: osdmap e160: 8 total, 7 up, 8 in 2024-09-13T21:52:01.887 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:01 smithi154 ceph-mon[65435]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T21:52:01.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 256740 9.59 MB/sec execute 554 sec latency 71.256 ms 2024-09-13T21:52:02.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:01 smithi181 podman[62959]: 2024-09-13 21:52:01.803565055 +0000 UTC m=+0.377036410 container init ee73f9df3842cca5f1b5e2857f02c344f8819d0749108d416e31adfae20d44bb (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-13T21:52:02.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:01 smithi181 podman[62959]: 2024-09-13 21:52:01.807717126 +0000 UTC m=+0.381188486 container start ee73f9df3842cca5f1b5e2857f02c344f8819d0749108d416e31adfae20d44bb (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T21:52:02.151 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:01 smithi181 podman[62959]: 2024-09-13 21:52:01.895457037 +0000 UTC m=+0.468928402 container attach ee73f9df3842cca5f1b5e2857f02c344f8819d0749108d416e31adfae20d44bb (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T21:52:02.151 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:02 smithi181 podman[62959]: 2024-09-13 21:52:02.035130389 +0000 UTC m=+0.608601752 container died ee73f9df3842cca5f1b5e2857f02c344f8819d0749108d416e31adfae20d44bb (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:52:02.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 257082 9.59 MB/sec execute 555 sec latency 35.765 ms 2024-09-13T21:52:03.057 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:02 smithi154 ceph-mon[65435]: Health check failed: Degraded data redundancy: 108/1581 objects degraded (6.831%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:03.079 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:02 smithi181 podman[62982]: 2024-09-13 21:52:02.817085534 +0000 UTC m=+0.772072250 container remove ee73f9df3842cca5f1b5e2857f02c344f8819d0749108d416e31adfae20d44bb (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T21:52:03.079 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:02 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-13T21:52:03.079 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:02 smithi181 systemd[1]: Stopped Ceph osd.7 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:52:03.080 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:02 smithi181 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 38.442s CPU time. 2024-09-13T21:52:03.080 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:02 smithi181 ceph-mon[51676]: Health check failed: Degraded data redundancy: 108/1581 objects degraded (6.831%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:03.400 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 systemd[1]: Starting Ceph osd.7 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:52:03.400 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 podman[63083]: 2024-09-13 21:52:03.218402986 +0000 UTC m=+0.019359415 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:52:03.400 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 podman[63083]: 2024-09-13 21:52:03.327431652 +0000 UTC m=+0.128388068 container create a6a17bfec0c30006eac34e2b846a6c65b9053661f87056e1b3aa04fc1d68c6ff (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:52:03.724 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 podman[63083]: 2024-09-13 21:52:03.564246902 +0000 UTC m=+0.365203327 container init a6a17bfec0c30006eac34e2b846a6c65b9053661f87056e1b3aa04fc1d68c6ff (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:52:03.724 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 podman[63083]: 2024-09-13 21:52:03.56879854 +0000 UTC m=+0.369754961 container start a6a17bfec0c30006eac34e2b846a6c65b9053661f87056e1b3aa04fc1d68c6ff (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:52:03.724 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 podman[63083]: 2024-09-13 21:52:03.656026322 +0000 UTC m=+0.456982746 container attach a6a17bfec0c30006eac34e2b846a6c65b9053661f87056e1b3aa04fc1d68c6ff (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T21:52:03.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:03 smithi154 ceph-mon[65435]: pgmap v297: 97 pgs: 20 peering, 2 stale+active+clean, 13 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.9 MiB/s wr, 395 op/s; 108/1581 objects degraded (6.831%) 2024-09-13T21:52:03.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 257385 9.59 MB/sec execute 556 sec latency 62.320 ms 2024-09-13T21:52:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:03 smithi181 ceph-mon[51676]: pgmap v297: 97 pgs: 20 peering, 2 stale+active+clean, 13 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.9 MiB/s wr, 395 op/s; 108/1581 objects degraded (6.831%) 2024-09-13T21:52:04.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:52:04.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 bash[63083]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:52:04.151 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:52:04.151 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:03 smithi181 bash[63083]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T21:52:04.513 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T21:52:04.513 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 bash[63083]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T21:52:04.513 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T21:52:04.513 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 bash[63083]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T21:52:04.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T21:52:04.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 bash[63083]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T21:52:04.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T21:52:04.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 bash[63083]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T21:52:04.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T21:52:04.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 bash[63083]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T21:52:04.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T21:52:04.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 bash[63083]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T21:52:04.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate[63129]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T21:52:04.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 bash[63083]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T21:52:04.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:04 smithi181 podman[63083]: 2024-09-13 21:52:04.550100233 +0000 UTC m=+1.351056658 container died a6a17bfec0c30006eac34e2b846a6c65b9053661f87056e1b3aa04fc1d68c6ff (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T21:52:04.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 257731 9.59 MB/sec execute 557 sec latency 63.555 ms 2024-09-13T21:52:05.917 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:05 smithi154 ceph-mon[65435]: pgmap v298: 97 pgs: 2 active+undersized, 20 peering, 19 active+undersized+degraded, 56 active+clean; 351 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s wr, 411 op/s; 148/1590 objects degraded (9.308%); 508 B/s, 0 objects/s recovering 2024-09-13T21:52:05.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 258117 9.59 MB/sec execute 558 sec latency 41.847 ms 2024-09-13T21:52:06.084 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:05 smithi181 ceph-mon[51676]: pgmap v298: 97 pgs: 2 active+undersized, 20 peering, 19 active+undersized+degraded, 56 active+clean; 351 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s wr, 411 op/s; 148/1590 objects degraded (9.308%); 508 B/s, 0 objects/s recovering 2024-09-13T21:52:06.084 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:05 smithi181 podman[63259]: 2024-09-13 21:52:05.736044697 +0000 UTC m=+1.176007005 container remove a6a17bfec0c30006eac34e2b846a6c65b9053661f87056e1b3aa04fc1d68c6ff (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T21:52:06.085 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:05 smithi181 podman[63315]: 2024-09-13 21:52:05.953775496 +0000 UTC m=+0.112908414 container create a55b1845224949f619dc4f5fd9aa189d8684326b7c1f530f9ac63e072da52c1a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-13T21:52:06.085 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:05 smithi181 podman[63315]: 2024-09-13 21:52:05.859801509 +0000 UTC m=+0.018934441 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:52:06.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:06 smithi181 podman[63315]: 2024-09-13 21:52:06.206580252 +0000 UTC m=+0.365713171 container init a55b1845224949f619dc4f5fd9aa189d8684326b7c1f530f9ac63e072da52c1a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T21:52:06.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:06 smithi181 podman[63315]: 2024-09-13 21:52:06.210224178 +0000 UTC m=+0.369357097 container start a55b1845224949f619dc4f5fd9aa189d8684326b7c1f530f9ac63e072da52c1a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:52:06.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:06 smithi181 ceph-osd[63335]: -- 172.21.15.181:0/1711905166 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e9f490a1e0 con 0x55e9f48e4000 2024-09-13T21:52:06.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:06 smithi181 bash[63315]: a55b1845224949f619dc4f5fd9aa189d8684326b7c1f530f9ac63e072da52c1a 2024-09-13T21:52:06.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:06 smithi181 systemd[1]: Started Ceph osd.7 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:52:06.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 258601 9.60 MB/sec execute 559 sec latency 32.565 ms 2024-09-13T21:52:07.493 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:07.493 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:07.493 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:07.494 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:07 smithi154 ceph-mon[65435]: pgmap v299: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.3 MiB/s wr, 338 op/s; 231/1590 objects degraded (14.528%); 508 B/s, 0 objects/s recovering 2024-09-13T21:52:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:07.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:07.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:07.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:07 smithi181 ceph-mon[51676]: pgmap v299: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.3 MiB/s wr, 338 op/s; 231/1590 objects degraded (14.528%); 508 B/s, 0 objects/s recovering 2024-09-13T21:52:07.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 259055 9.61 MB/sec execute 560 sec latency 28.810 ms 2024-09-13T21:52:08.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:07 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:52:07.847+0000 7efe2c51e640 -1 Falling back to public interface 2024-09-13T21:52:08.520 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:08 smithi154 ceph-mon[65435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-13T21:52:08.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:08 smithi181 ceph-mon[51676]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-13T21:52:08.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 259401 9.61 MB/sec execute 561 sec latency 50.437 ms 2024-09-13T21:52:09.545 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:09 smithi154 ceph-mon[65435]: pgmap v300: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.2 MiB/s wr, 265 op/s; 247/1641 objects degraded (15.052%); 458 B/s, 0 objects/s recovering 2024-09-13T21:52:09.649 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:09 smithi181 ceph-mon[51676]: pgmap v300: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.2 MiB/s wr, 265 op/s; 247/1641 objects degraded (15.052%); 458 B/s, 0 objects/s recovering 2024-09-13T21:52:09.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 259690 9.61 MB/sec execute 562 sec latency 60.649 ms 2024-09-13T21:52:10.557 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:10 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 247/1641 objects degraded (15.052%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:10.557 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:10.557 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:10 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 247/1641 objects degraded (15.052%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:10.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:10.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 260018 9.60 MB/sec execute 563 sec latency 56.119 ms 2024-09-13T21:52:11.580 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:11.580 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:52:11.580 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:11 smithi154 ceph-mon[65435]: pgmap v301: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 453 op/s; 258/1686 objects degraded (15.302%); 406 B/s, 0 objects/s recovering 2024-09-13T21:52:11.581 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:11.581 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:52:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:11 smithi181 ceph-mon[51676]: pgmap v301: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 453 op/s; 258/1686 objects degraded (15.302%); 406 B/s, 0 objects/s recovering 2024-09-13T21:52:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:11.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 260267 9.59 MB/sec execute 564 sec latency 67.339 ms 2024-09-13T21:52:12.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 260625 9.59 MB/sec execute 565 sec latency 59.095 ms 2024-09-13T21:52:13.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:13 smithi154 ceph-mon[65435]: pgmap v302: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 415 op/s; 248/1674 objects degraded (14.815%); 343 B/s, 0 objects/s recovering 2024-09-13T21:52:13.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 260972 9.59 MB/sec execute 566 sec latency 43.770 ms 2024-09-13T21:52:14.092 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:13 smithi181 ceph-mon[51676]: pgmap v302: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 415 op/s; 248/1674 objects degraded (14.815%); 343 B/s, 0 objects/s recovering 2024-09-13T21:52:14.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 261318 9.59 MB/sec execute 567 sec latency 40.290 ms 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: pgmap v303: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.6 MiB/s wr, 469 op/s; 252/1668 objects degraded (15.108%); 338 B/s, 0 objects/s recovering 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:15.667 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all osd 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-13T21:52:15.668 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T21:52:15.669 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:15 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-13T21:52:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: pgmap v303: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.6 MiB/s wr, 469 op/s; 252/1668 objects degraded (15.108%); 338 B/s, 0 objects/s recovering 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all osd 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-13T21:52:15.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T21:52:15.902 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:15 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-13T21:52:15.947 INFO:teuthology.orchestra.run.smithi154.stdout: 5 261674 9.60 MB/sec execute 568 sec latency 55.628 ms 2024-09-13T21:52:16.684 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:16 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 252/1668 objects degraded (15.108%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:16.684 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:16 smithi154 ceph-mon[65435]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T21:52:16.684 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:16 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T21:52:16.684 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:16 smithi154 ceph-mon[65435]: osdmap e161: 8 total, 7 up, 8 in 2024-09-13T21:52:16.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:16 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi154.uynwpr"]}]: dispatch 2024-09-13T21:52:16.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:16 smithi154 ceph-mon[65435]: Upgrade: It appears safe to stop mds.foofs.smithi154.uynwpr 2024-09-13T21:52:16.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:16 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 252/1668 objects degraded (15.108%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:16 smithi181 ceph-mon[51676]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T21:52:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:16 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T21:52:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:16 smithi181 ceph-mon[51676]: osdmap e161: 8 total, 7 up, 8 in 2024-09-13T21:52:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:16 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi154.uynwpr"]}]: dispatch 2024-09-13T21:52:16.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:16 smithi181 ceph-mon[51676]: Upgrade: It appears safe to stop mds.foofs.smithi154.uynwpr 2024-09-13T21:52:16.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 261963 9.59 MB/sec execute 569 sec latency 65.803 ms 2024-09-13T21:52:17.693 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:17 smithi154 ceph-mon[65435]: pgmap v305: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 604 op/s; 251/1635 objects degraded (15.352%) 2024-09-13T21:52:17.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:17 smithi181 ceph-mon[51676]: pgmap v305: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 604 op/s; 251/1635 objects degraded (15.352%) 2024-09-13T21:52:17.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:17 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:52:17.600+0000 7efe2c51e640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-13T21:52:17.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 262312 9.59 MB/sec execute 570 sec latency 53.868 ms 2024-09-13T21:52:18.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 262613 9.59 MB/sec execute 571 sec latency 43.616 ms 2024-09-13T21:52:19.400 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:18 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:52:18.922+0000 7efe2c51e640 -1 osd.7 158 log_to_monitors true 2024-09-13T21:52:19.720 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:19 smithi154 ceph-mon[65435]: pgmap v306: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 602 op/s; 254/1632 objects degraded (15.564%) 2024-09-13T21:52:19.720 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:19 smithi154 ceph-mon[65435]: Upgrade: Updating mds.foofs.smithi154.uynwpr 2024-09-13T21:52:19.720 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:19.721 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi154.uynwpr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:52:19.721 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:19.721 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:19 smithi154 ceph-mon[65435]: Deploying daemon mds.foofs.smithi154.uynwpr on smithi154 2024-09-13T21:52:19.721 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:19 smithi154 ceph-mon[65435]: from='osd.7 [v2:172.21.15.181:6824/4271645428,v1:172.21.15.181:6825/4271645428]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:52:19.721 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:19 smithi154 ceph-mon[65435]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:52:19.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:52:19.677+0000 7efe242c9640 -1 osd.7 158 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T21:52:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-mon[51676]: pgmap v306: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 602 op/s; 254/1632 objects degraded (15.564%) 2024-09-13T21:52:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-mon[51676]: Upgrade: Updating mds.foofs.smithi154.uynwpr 2024-09-13T21:52:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi154.uynwpr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:52:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-mon[51676]: Deploying daemon mds.foofs.smithi154.uynwpr on smithi154 2024-09-13T21:52:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-mon[51676]: from='osd.7 [v2:172.21.15.181:6824/4271645428,v1:172.21.15.181:6825/4271645428]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:52:19.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:19 smithi181 ceph-mon[51676]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T21:52:19.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263022 9.60 MB/sec execute 572 sec latency 42.186 ms 2024-09-13T21:52:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:20 smithi154 ceph-mon[65435]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T21:52:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:20 smithi154 ceph-mon[65435]: from='osd.7 [v2:172.21.15.181:6824/4271645428,v1:172.21.15.181:6825/4271645428]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:52:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:20 smithi154 ceph-mon[65435]: osdmap e162: 8 total, 7 up, 8 in 2024-09-13T21:52:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:20 smithi154 ceph-mon[65435]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:52:20.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:20 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 254/1632 objects degraded (15.564%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:20.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263349 9.59 MB/sec execute 573 sec latency 299.541 ms 2024-09-13T21:52:21.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:20 smithi181 ceph-mon[51676]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T21:52:21.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:20 smithi181 ceph-mon[51676]: from='osd.7 [v2:172.21.15.181:6824/4271645428,v1:172.21.15.181:6825/4271645428]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:52:21.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:20 smithi181 ceph-mon[51676]: osdmap e162: 8 total, 7 up, 8 in 2024-09-13T21:52:21.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:20 smithi181 ceph-mon[51676]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-09-13T21:52:21.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:20 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 254/1632 objects degraded (15.564%), 34 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:21.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:21 smithi154 ceph-mon[65435]: pgmap v308: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 356 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.0 MiB/s wr, 526 op/s; 267/1683 objects degraded (15.865%) 2024-09-13T21:52:21.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:21 smithi154 ceph-mon[65435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:52:21.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:21 smithi154 ceph-mon[65435]: osd.7 [v2:172.21.15.181:6824/4271645428,v1:172.21.15.181:6825/4271645428] boot 2024-09-13T21:52:21.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:21 smithi154 ceph-mon[65435]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T21:52:21.783 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:21 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:52:21.784 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:21 smithi154 ceph-mon[65435]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T21:52:21.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263349 9.58 MB/sec execute 574 sec latency 1299.568 ms 2024-09-13T21:52:22.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:21 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[65431]: 2024-09-13T21:52:21.780+0000 7f7c1a129640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:52:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-mon[51676]: pgmap v308: 97 pgs: 7 active+undersized, 34 active+undersized+degraded, 56 active+clean; 356 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.0 MiB/s wr, 526 op/s; 267/1683 objects degraded (15.865%) 2024-09-13T21:52:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-mon[51676]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T21:52:22.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-mon[51676]: osd.7 [v2:172.21.15.181:6824/4271645428,v1:172.21.15.181:6825/4271645428] boot 2024-09-13T21:52:22.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-mon[51676]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T21:52:22.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T21:52:22.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-mon[51676]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T21:52:22.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:52:21.897+0000 7fae4166f640 -1 bdev(0x55939aa26c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 6 2024-09-13T21:52:22.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:52:21.899+0000 7fae40e6e640 -1 bdev(0x55939aa26c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 13 2024-09-13T21:52:22.151 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:52:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:52:21.901+0000 7fae3e669640 -1 bdev(0x55939aa26c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 18 2024-09-13T21:52:22.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263349 9.56 MB/sec execute 575 sec latency 2299.622 ms 2024-09-13T21:52:23.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:22 smithi154 ceph-mon[65435]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T21:52:23.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:22 smithi154 ceph-mon[65435]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:52:23.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:22 smithi154 ceph-mon[65435]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T21:52:23.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:22 smithi154 ceph-mon[65435]: Standby daemon mds.foofs.smithi181.mgvpzf assigned to filesystem foofs as rank 0 2024-09-13T21:52:23.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:22 smithi154 ceph-mon[65435]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T21:52:23.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:22 smithi154 ceph-mon[65435]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T21:52:23.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:22 smithi154 ceph-mon[65435]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T21:52:23.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:22 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:replay} 2024-09-13T21:52:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:22 smithi181 ceph-mon[51676]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T21:52:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:22 smithi181 ceph-mon[51676]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:52:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:22 smithi181 ceph-mon[51676]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T21:52:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:22 smithi181 ceph-mon[51676]: Standby daemon mds.foofs.smithi181.mgvpzf assigned to filesystem foofs as rank 0 2024-09-13T21:52:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:22 smithi181 ceph-mon[51676]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T21:52:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:22 smithi181 ceph-mon[51676]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T21:52:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:22 smithi181 ceph-mon[51676]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T21:52:23.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:22 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:replay} 2024-09-13T21:52:23.874 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:52:23.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263349 9.54 MB/sec execute 576 sec latency 3299.651 ms 2024-09-13T21:52:24.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:23 smithi154 ceph-mon[65435]: pgmap v312: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 remapped+peering, 3 peering, 7 active+undersized, 24 active+undersized+degraded, 58 active+clean; 355 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 6.0 MiB/s wr, 496 op/s; 213/1743 objects degraded (12.220%); 1.1 MiB/s, 8 objects/s recovering 2024-09-13T21:52:24.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:23 smithi154 ceph-mon[65435]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T21:52:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:23 smithi181 ceph-mon[51676]: pgmap v312: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 remapped+peering, 3 peering, 7 active+undersized, 24 active+undersized+degraded, 58 active+clean; 355 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 MiB/s rd, 6.0 MiB/s wr, 496 op/s; 213/1743 objects degraded (12.220%); 1.1 MiB/s, 8 objects/s recovering 2024-09-13T21:52:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:23 smithi181 ceph-mon[51676]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (9m) 2m ago 18m 27.0M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (6m) 2m ago 17m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (6m) 14s ago 17m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (9m) 2m ago 17m 52.7M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (12m) 2m ago 12m 37.3M - 17.2.0 e1d6a67b021e 3935c374da79 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (12m) 14s ago 12m 18.0M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (8m) 2m ago 19m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (7m) 14s ago 17m 443M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:52:24.554 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (7m) 2m ago 19m 50.1M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (6m) 14s ago 17m 45.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (11m) 2m ago 11m 279M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (11m) 14s ago 11m 84.8M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (9m) 2m ago 17m 15.4M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (8m) 14s ago 17m 15.7M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (5m) 2m ago 16m 333M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (4m) 2m ago 16m 257M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (3m) 2m ago 15m 201M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (3m) 2m ago 15m 14.2M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (2m) 14s ago 14m 349M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (107s) 14s ago 14m 325M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 80819532f2ce 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (70s) 14s ago 14m 308M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 07e99a632a76 2024-09-13T21:52:24.555 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (18s) 14s ago 13m 11.0M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 a55b18452249 2024-09-13T21:52:24.556 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (7m) 2m ago 16m 66.1M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:52:24.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263349 9.53 MB/sec execute 577 sec latency 4299.710 ms 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:52:24.973 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:52:24.974 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T21:52:24.974 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:52:24.974 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:52:24.974 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T21:52:24.974 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 12 2024-09-13T21:52:24.974 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:52:24.974 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:52:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:24 smithi181 ceph-mon[51676]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T21:52:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:24 smithi181 ceph-mon[51676]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T21:52:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:24 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.154:6834/4085008546,v1:172.21.15.154:6835/4085008546] up:boot 2024-09-13T21:52:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:24 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:replay} 1 up:standby 2024-09-13T21:52:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:24 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:52:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:24 smithi181 ceph-mon[51676]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:24 smithi181 ceph-mon[51676]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T21:52:25.319 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:52:25.319 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:52:25.319 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:52:25.319 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:52:25.319 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:52:25.319 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:52:25.319 INFO:teuthology.orchestra.run.smithi154.stdout: "osd", 2024-09-13T21:52:25.319 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:52:25.320 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:52:25.320 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:52:25.320 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "14/23 daemons upgraded", 2024-09-13T21:52:25.320 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading mds daemons", 2024-09-13T21:52:25.320 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:52:25.320 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:52:25.320 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:24 smithi154 ceph-mon[65435]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T21:52:25.320 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:24 smithi154 ceph-mon[65435]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T21:52:25.320 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:24 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.154:6834/4085008546,v1:172.21.15.154:6835/4085008546] up:boot 2024-09-13T21:52:25.320 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:24 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:replay} 1 up:standby 2024-09-13T21:52:25.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:24 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:52:25.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:24 smithi154 ceph-mon[65435]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:25.321 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:24 smithi154 ceph-mon[65435]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T21:52:25.743 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_WARN 1 filesystem is degraded; Degraded data redundancy: 181/1743 objects degraded (10.384%), 21 pgs degraded 2024-09-13T21:52:25.743 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-09-13T21:52:25.743 INFO:teuthology.orchestra.run.smithi154.stdout: fs foofs is degraded 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 181/1743 objects degraded (10.384%), 21 pgs degraded 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.8 is active+undersized+degraded, acting [6,1] 2024-09-13T21:52:25.744 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.13 is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-09-13T21:52:25.745 INFO:teuthology.orchestra.run.smithi154.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-09-13T21:52:25.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263349 9.51 MB/sec execute 578 sec latency 5299.777 ms 2024-09-13T21:52:26.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:26.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: pgmap v315: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 remapped+peering, 3 peering, 5 active+undersized, 20 active+undersized+degraded, 64 active+clean; 355 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 226 op/s; 181/1743 objects degraded (10.384%); 1.7 MiB/s, 15 objects/s recovering 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: reconnect by client.24333 172.21.15.181:0/2693092852 after 0.000999981 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: reconnect by client.14556 172.21.15.154:0/4061411931 after 0.000999981 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:reconnect 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:reconnect} 1 up:standby 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/236289899' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: Health check update: Degraded data redundancy: 181/1743 objects degraded (10.384%), 21 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:26.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:25 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2059329657' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: pgmap v315: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 2 remapped+peering, 3 peering, 5 active+undersized, 20 active+undersized+degraded, 64 active+clean; 355 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.0 MiB/s wr, 226 op/s; 181/1743 objects degraded (10.384%); 1.7 MiB/s, 15 objects/s recovering 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: reconnect by client.24333 172.21.15.181:0/2693092852 after 0.000999981 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: reconnect by client.14556 172.21.15.154:0/4061411931 after 0.000999981 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:reconnect 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:reconnect} 1 up:standby 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/236289899' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:52:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: Health check update: Degraded data redundancy: 181/1743 objects degraded (10.384%), 21 pgs degraded (PG_DEGRADED) 2024-09-13T21:52:26.151 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:25 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2059329657' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:52:26.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263349 9.49 MB/sec execute 579 sec latency 6299.812 ms 2024-09-13T21:52:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:26 smithi181 ceph-mon[51676]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:26 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:rejoin 2024-09-13T21:52:27.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:26 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:rejoin} 1 up:standby 2024-09-13T21:52:27.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:26 smithi154 ceph-mon[65435]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:27.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:26 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:rejoin 2024-09-13T21:52:27.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:26 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:rejoin} 1 up:standby 2024-09-13T21:52:27.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263368 9.48 MB/sec execute 580 sec latency 7220.878 ms 2024-09-13T21:52:28.115 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:27 smithi154 ceph-mon[65435]: pgmap v317: 97 pgs: 1 active+undersized+remapped, 1 remapped+peering, 5 peering, 90 active+clean; 356 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 1.2 MiB/s wr, 116 op/s; 3.2 MiB/s, 0 keys/s, 35 objects/s recovering 2024-09-13T21:52:28.115 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:27 smithi154 ceph-mon[65435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 181/1743 objects degraded (10.384%), 21 pgs degraded) 2024-09-13T21:52:28.115 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:27 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:clientreplay 2024-09-13T21:52:28.115 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:27 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:clientreplay} 1 up:standby 2024-09-13T21:52:28.116 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:27 smithi154 ceph-mon[65435]: daemon mds.foofs.smithi181.mgvpzf is now active in filesystem foofs as rank 0 2024-09-13T21:52:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:27 smithi181 ceph-mon[51676]: pgmap v317: 97 pgs: 1 active+undersized+remapped, 1 remapped+peering, 5 peering, 90 active+clean; 356 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 1.2 MiB/s wr, 116 op/s; 3.2 MiB/s, 0 keys/s, 35 objects/s recovering 2024-09-13T21:52:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:27 smithi181 ceph-mon[51676]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 181/1743 objects degraded (10.384%), 21 pgs degraded) 2024-09-13T21:52:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:27 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:clientreplay 2024-09-13T21:52:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:27 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi181.mgvpzf=up:clientreplay} 1 up:standby 2024-09-13T21:52:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:27 smithi181 ceph-mon[51676]: daemon mds.foofs.smithi181.mgvpzf is now active in filesystem foofs as rank 0 2024-09-13T21:52:28.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:28 smithi154 ceph-mon[65435]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T21:52:28.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:28 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:52:28.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:28 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:active 2024-09-13T21:52:28.874 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:28 smithi154 ceph-mon[65435]: fsmap foofs:1 {0=foofs.smithi181.mgvpzf=up:active} 1 up:standby 2024-09-13T21:52:28.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263542 9.47 MB/sec execute 581 sec latency 111.540 ms 2024-09-13T21:52:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:28 smithi181 ceph-mon[51676]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T21:52:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:28 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:52:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:28 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.181:6832/2102910073,v1:172.21.15.181:6833/2102910073] up:active 2024-09-13T21:52:29.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:28 smithi181 ceph-mon[51676]: fsmap foofs:1 {0=foofs.smithi181.mgvpzf=up:active} 1 up:standby 2024-09-13T21:52:29.901 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:29 smithi154 ceph-mon[65435]: pgmap v318: 97 pgs: 5 peering, 92 active+clean; 356 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 640 KiB/s wr, 87 op/s; 2.0 MiB/s, 0 keys/s, 26 objects/s recovering 2024-09-13T21:52:29.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263710 9.46 MB/sec execute 582 sec latency 100.610 ms 2024-09-13T21:52:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:29 smithi181 ceph-mon[51676]: pgmap v318: 97 pgs: 5 peering, 92 active+clean; 356 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 640 KiB/s wr, 87 op/s; 2.0 MiB/s, 0 keys/s, 26 objects/s recovering 2024-09-13T21:52:30.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263860 9.44 MB/sec execute 583 sec latency 111.724 ms 2024-09-13T21:52:31.948 INFO:teuthology.orchestra.run.smithi154.stdout: 5 263964 9.43 MB/sec execute 584 sec latency 138.750 ms 2024-09-13T21:52:31.987 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:31 smithi154 ceph-mon[65435]: pgmap v319: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 807 KiB/s wr, 101 op/s; 2.2 MiB/s, 0 keys/s, 36 objects/s recovering 2024-09-13T21:52:32.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:31 smithi181 ceph-mon[51676]: pgmap v319: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 807 KiB/s wr, 101 op/s; 2.2 MiB/s, 0 keys/s, 36 objects/s recovering 2024-09-13T21:52:32.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 264113 9.43 MB/sec execute 585 sec latency 146.339 ms 2024-09-13T21:52:33.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 264226 9.42 MB/sec execute 586 sec latency 85.677 ms 2024-09-13T21:52:34.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:33 smithi154 ceph-mon[65435]: pgmap v320: 97 pgs: 97 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.1 MiB/s wr, 173 op/s; 1.9 MiB/s, 0 keys/s, 32 objects/s recovering 2024-09-13T21:52:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:33 smithi181 ceph-mon[51676]: pgmap v320: 97 pgs: 97 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.1 MiB/s wr, 173 op/s; 1.9 MiB/s, 0 keys/s, 32 objects/s recovering 2024-09-13T21:52:34.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 264470 9.40 MB/sec execute 587 sec latency 72.291 ms 2024-09-13T21:52:35.132 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:34 smithi154 ceph-mon[65435]: daemon mds.foofs.smithi154.uynwpr restarted 2024-09-13T21:52:35.132 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:34 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:35.132 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:34 smithi154 ceph-mon[65435]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T21:52:35.133 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:34 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:35.133 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:34 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:35.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:34 smithi181 ceph-mon[51676]: daemon mds.foofs.smithi154.uynwpr restarted 2024-09-13T21:52:35.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:34 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:34 smithi181 ceph-mon[51676]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T21:52:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:34 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:35.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:34 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:35.913 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:35 smithi154 ceph-mon[65435]: pgmap v322: 97 pgs: 97 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 704 KiB/s wr, 117 op/s; 1.1 MiB/s, 24 objects/s recovering 2024-09-13T21:52:35.914 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:35 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:boot 2024-09-13T21:52:35.914 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:35 smithi154 ceph-mon[65435]: fsmap foofs:1 {0=foofs.smithi181.mgvpzf=up:active} 1 up:standby 2024-09-13T21:52:35.914 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:35 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:52:35.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 264687 9.41 MB/sec execute 588 sec latency 80.852 ms 2024-09-13T21:52:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:35 smithi181 ceph-mon[51676]: pgmap v322: 97 pgs: 97 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 704 KiB/s wr, 117 op/s; 1.1 MiB/s, 24 objects/s recovering 2024-09-13T21:52:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:35 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:boot 2024-09-13T21:52:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:35 smithi181 ceph-mon[51676]: fsmap foofs:1 {0=foofs.smithi181.mgvpzf=up:active} 1 up:standby 2024-09-13T21:52:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:35 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:52:36.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 264864 9.40 MB/sec execute 589 sec latency 141.611 ms 2024-09-13T21:52:37.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 265061 9.39 MB/sec execute 590 sec latency 88.465 ms 2024-09-13T21:52:38.015 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:37 smithi154 ceph-mon[65435]: pgmap v323: 97 pgs: 97 active+clean; 356 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.6 MiB/s wr, 204 op/s; 600 KiB/s, 15 objects/s recovering 2024-09-13T21:52:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:37 smithi181 ceph-mon[51676]: pgmap v323: 97 pgs: 97 active+clean; 356 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.6 MiB/s wr, 204 op/s; 600 KiB/s, 15 objects/s recovering 2024-09-13T21:52:38.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 265291 9.39 MB/sec execute 591 sec latency 77.021 ms 2024-09-13T21:52:39.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 265422 9.38 MB/sec execute 592 sec latency 87.687 ms 2024-09-13T21:52:40.065 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:39 smithi154 ceph-mon[65435]: pgmap v324: 97 pgs: 97 active+clean; 356 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 207 op/s; 509 KiB/s, 12 objects/s recovering 2024-09-13T21:52:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:39 smithi181 ceph-mon[51676]: pgmap v324: 97 pgs: 97 active+clean; 356 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 207 op/s; 509 KiB/s, 12 objects/s recovering 2024-09-13T21:52:40.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 265597 9.37 MB/sec execute 593 sec latency 107.514 ms 2024-09-13T21:52:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:52:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:41 smithi154 ceph-mon[65435]: pgmap v325: 97 pgs: 97 active+clean; 352 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.5 MiB/s wr, 295 op/s 2024-09-13T21:52:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:41.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:41.899 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:52:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:41 smithi181 ceph-mon[51676]: pgmap v325: 97 pgs: 97 active+clean; 352 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.5 MiB/s wr, 295 op/s 2024-09-13T21:52:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:41.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:41.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 265761 9.36 MB/sec execute 594 sec latency 92.289 ms 2024-09-13T21:52:42.949 INFO:teuthology.orchestra.run.smithi154.stdout: 5 265934 9.36 MB/sec execute 595 sec latency 98.722 ms 2024-09-13T21:52:43.151 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:43 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:43.151 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:43 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:43.151 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:43 smithi154 ceph-mon[65435]: pgmap v326: 97 pgs: 97 active+clean; 352 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 2.4 MiB/s wr, 249 op/s 2024-09-13T21:52:43.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:43 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:43.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:43 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:43.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:43 smithi181 ceph-mon[51676]: pgmap v326: 97 pgs: 97 active+clean; 352 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 2.4 MiB/s wr, 249 op/s 2024-09-13T21:52:43.950 INFO:teuthology.orchestra.run.smithi154.stdout: 5 266049 9.34 MB/sec execute 596 sec latency 102.104 ms 2024-09-13T21:52:44.950 INFO:teuthology.orchestra.run.smithi154.stdout: 5 266186 9.33 MB/sec execute 597 sec latency 119.310 ms 2024-09-13T21:52:45.795 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:45 smithi154 ceph-mon[65435]: pgmap v327: 97 pgs: 97 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 2.6 MiB/s wr, 254 op/s 2024-09-13T21:52:45.950 INFO:teuthology.orchestra.run.smithi154.stdout: 5 266339 9.33 MB/sec execute 598 sec latency 87.647 ms 2024-09-13T21:52:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:45 smithi181 ceph-mon[51676]: pgmap v327: 97 pgs: 97 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 2.6 MiB/s wr, 254 op/s 2024-09-13T21:52:46.950 INFO:teuthology.orchestra.run.smithi154.stdout: 5 266483 9.32 MB/sec execute 599 sec latency 98.445 ms 2024-09-13T21:52:47.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: pgmap v328: 97 pgs: 97 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 2.9 MiB/s wr, 266 op/s 2024-09-13T21:52:47.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:47.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:47.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:47.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:47.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:47.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:47.881 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:47.882 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:47.882 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:47.882 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:47.882 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:47.882 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:47 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi181.mgvpzf"]}]: dispatch 2024-09-13T21:52:47.950 INFO:teuthology.orchestra.run.smithi154.stdout: 5 cleanup 600 sec 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: pgmap v328: 97 pgs: 97 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 2.9 MiB/s wr, 266 op/s 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:48.018 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:48.019 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:48.019 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:47 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi181.mgvpzf"]}]: dispatch 2024-09-13T21:52:48.925 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:48 smithi154 ceph-mon[65435]: Detected new or changed devices on smithi154 2024-09-13T21:52:48.925 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:48 smithi154 ceph-mon[65435]: Upgrade: It appears safe to stop mds.foofs.smithi181.mgvpzf 2024-09-13T21:52:48.950 INFO:teuthology.orchestra.run.smithi154.stdout: 5 cleanup 601 sec 2024-09-13T21:52:49.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:48 smithi181 ceph-mon[51676]: Detected new or changed devices on smithi154 2024-09-13T21:52:49.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:48 smithi181 ceph-mon[51676]: Upgrade: It appears safe to stop mds.foofs.smithi181.mgvpzf 2024-09-13T21:52:49.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:49 smithi154 ceph-mon[65435]: pgmap v329: 97 pgs: 97 active+clean; 352 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.4 MiB/s wr, 245 op/s 2024-09-13T21:52:49.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:49 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:49.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:49 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.mgvpzf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:52:49.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:49 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:49.950 INFO:teuthology.orchestra.run.smithi154.stdout: 1 cleanup 602 sec 2024-09-13T21:52:49.958 INFO:teuthology.orchestra.run.smithi154.stdout: 0 cleanup 602 sec 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Operation Count AvgLat MaxLat 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: ---------------------------------------- 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: NTCreateX 178930 4.778 7220.875 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Close 131297 0.136 10.775 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Rename 7590 9.731 1066.467 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Unlink 36229 7.058 7217.746 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Qpathinfo 162177 1.248 1024.568 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Qfileinfo 28269 0.007 2.920 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Qfsinfo 29818 0.876 1497.283 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Sfileinfo 14563 13.489 7219.949 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: Find 62719 0.983 61.937 2024-09-13T21:52:49.959 INFO:teuthology.orchestra.run.smithi154.stdout: WriteX 88406 14.756 7219.878 2024-09-13T21:52:49.960 INFO:teuthology.orchestra.run.smithi154.stdout: ReadX 281150 0.010 2.553 2024-09-13T21:52:49.960 INFO:teuthology.orchestra.run.smithi154.stdout: LockX 586 0.582 20.620 2024-09-13T21:52:49.960 INFO:teuthology.orchestra.run.smithi154.stdout: UnlockX 586 0.441 11.161 2024-09-13T21:52:49.960 INFO:teuthology.orchestra.run.smithi154.stdout: Flush 12548 0.002 0.046 2024-09-13T21:52:49.960 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:52:49.960 INFO:teuthology.orchestra.run.smithi154.stdout:Throughput 9.31832 MB/sec 5 clients 5 procs max_latency=7220.878 ms 2024-09-13T21:52:49.961 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-13T21:52:49.998 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-13T21:52:50.003 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:49 smithi181 ceph-mon[51676]: pgmap v329: 97 pgs: 97 active+clean; 352 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.4 MiB/s wr, 245 op/s 2024-09-13T21:52:50.004 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:49 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:50.004 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:49 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.mgvpzf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T21:52:50.004 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:49 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:52:50.117 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-13T21:52:50.142 INFO:teuthology.orchestra.run.smithi154.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-13T21:52:50.142 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:52:50.142 INFO:teuthology.orchestra.run.smithi154.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-13T21:52:50.142 INFO:teuthology.orchestra.run.smithi154.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-13T21:52:50.243 INFO:teuthology.orchestra.run.smithi154.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-13T21:52:50.243 INFO:teuthology.orchestra.run.smithi154.stdout:releasing clients 2024-09-13T21:52:50.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:50 smithi154 ceph-mon[65435]: Upgrade: Updating mds.foofs.smithi181.mgvpzf 2024-09-13T21:52:50.772 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:50 smithi154 ceph-mon[65435]: Deploying daemon mds.foofs.smithi181.mgvpzf on smithi181 2024-09-13T21:52:51.003 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:50 smithi181 ceph-mon[51676]: Upgrade: Updating mds.foofs.smithi181.mgvpzf 2024-09-13T21:52:51.004 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:50 smithi181 ceph-mon[51676]: Deploying daemon mds.foofs.smithi181.mgvpzf on smithi181 2024-09-13T21:52:51.322 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[65431]: 2024-09-13T21:52:51.287+0000 7f7c1a129640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:52:51.650 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:52:51.383+0000 7fae3ce66640 -1 bdev(0x55939aa26c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 1 2024-09-13T21:52:51.650 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:52:51.384+0000 7fae3d667640 -1 bdev(0x55939aa26c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-13T21:52:51.946 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: pgmap v330: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.8 MiB/s wr, 264 op/s 2024-09-13T21:52:51.946 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T21:52:51.946 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:52:51.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T21:52:51.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: Standby daemon mds.foofs.smithi154.uynwpr assigned to filesystem foofs as rank 0 2024-09-13T21:52:51.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T21:52:51.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T21:52:51.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T21:52:51.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi154.uynwpr=up:replay} 2024-09-13T21:52:51.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: pgmap v330: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.8 MiB/s wr, 264 op/s 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: Standby daemon mds.foofs.smithi154.uynwpr assigned to filesystem foofs as rank 0 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi154.uynwpr=up:replay} 2024-09-13T21:52:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:52:52.243 INFO:teuthology.orchestra.run.smithi154.stdout: 5 109 1.05 MB/sec execute 1 sec latency 1032.013 ms 2024-09-13T21:52:53.243 INFO:teuthology.orchestra.run.smithi154.stdout: 5 109 0.52 MB/sec execute 2 sec latency 2032.071 ms 2024-09-13T21:52:54.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:53 smithi154 ceph-mon[65435]: pgmap v332: 97 pgs: 97 active+clean; 344 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.7 MiB/s wr, 195 op/s 2024-09-13T21:52:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:53 smithi181 ceph-mon[51676]: pgmap v332: 97 pgs: 97 active+clean; 344 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 2.7 MiB/s wr, 195 op/s 2024-09-13T21:52:54.243 INFO:teuthology.orchestra.run.smithi154.stdout: 5 109 0.35 MB/sec execute 3 sec latency 3032.128 ms 2024-09-13T21:52:55.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:54 smithi154 ceph-mon[65435]: reconnect by client.14556 172.21.15.154:0/4061411931 after 0 2024-09-13T21:52:55.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:54 smithi154 ceph-mon[65435]: reconnect by client.24333 172.21.15.181:0/2693092852 after 0.000999973 2024-09-13T21:52:55.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:54 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:reconnect 2024-09-13T21:52:55.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:54 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi154.uynwpr=up:reconnect} 2024-09-13T21:52:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:54 smithi181 ceph-mon[51676]: reconnect by client.14556 172.21.15.154:0/4061411931 after 0 2024-09-13T21:52:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:54 smithi181 ceph-mon[51676]: reconnect by client.24333 172.21.15.181:0/2693092852 after 0.000999973 2024-09-13T21:52:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:54 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:reconnect 2024-09-13T21:52:55.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:54 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi154.uynwpr=up:reconnect} 2024-09-13T21:52:55.243 INFO:teuthology.orchestra.run.smithi154.stdout: 5 109 0.26 MB/sec execute 4 sec latency 4032.178 ms 2024-09-13T21:52:56.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:55 smithi154 ceph-mon[65435]: pgmap v333: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 191 op/s 2024-09-13T21:52:56.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:55 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:rejoin 2024-09-13T21:52:56.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:55 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi154.uynwpr=up:rejoin} 2024-09-13T21:52:56.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:56.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:52:56.134 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:52:56.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:55 smithi181 ceph-mon[51676]: pgmap v333: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 2.4 MiB/s wr, 191 op/s 2024-09-13T21:52:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:55 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:rejoin 2024-09-13T21:52:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:55 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi154.uynwpr=up:rejoin} 2024-09-13T21:52:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:52:56.243 INFO:teuthology.orchestra.run.smithi154.stdout: 5 cleanup 5 sec 2024-09-13T21:52:56.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:56 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:clientreplay 2024-09-13T21:52:56.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:56 smithi154 ceph-mon[65435]: fsmap foofs:1/1 {0=foofs.smithi154.uynwpr=up:clientreplay} 2024-09-13T21:52:56.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:56 smithi154 ceph-mon[65435]: daemon mds.foofs.smithi154.uynwpr is now active in filesystem foofs as rank 0 2024-09-13T21:52:56.793 INFO:teuthology.orchestra.run.smithi154.stdout: 0 cleanup 6 sec 2024-09-13T21:52:56.793 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:52:56.793 INFO:teuthology.orchestra.run.smithi154.stdout: Operation Count AvgLat MaxLat 2024-09-13T21:52:56.793 INFO:teuthology.orchestra.run.smithi154.stdout: ---------------------------------------- 2024-09-13T21:52:56.793 INFO:teuthology.orchestra.run.smithi154.stdout: NTCreateX 6 1853.023 5506.286 2024-09-13T21:52:56.794 INFO:teuthology.orchestra.run.smithi154.stdout: Close 2 31.616 63.231 2024-09-13T21:52:56.794 INFO:teuthology.orchestra.run.smithi154.stdout: Qfileinfo 4 4092.422 5451.036 2024-09-13T21:52:56.794 INFO:teuthology.orchestra.run.smithi154.stdout: WriteX 19 0.021 0.037 2024-09-13T21:52:56.794 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:52:56.794 INFO:teuthology.orchestra.run.smithi154.stdout:Throughput 0.262133 MB/sec 5 clients 5 procs max_latency=4032.178 ms 2024-09-13T21:52:56.796 DEBUG:teuthology.parallel:result is None 2024-09-13T21:52:56.821 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:52:56.821 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (9m) 15s ago 18m 27.1M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:52:56.821 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (6m) 15s ago 18m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:52:56.821 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (6m) 46s ago 18m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:52:56.822 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (9m) 15s ago 18m 54.6M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:52:56.822 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (23s) 15s ago 12m 12.0M - 19.1.1-324-gecab7be5 0652ff2b5d78 e190f464195a 2024-09-13T21:52:56.822 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (12m) 46s ago 12m 18.0M - 17.2.0 e1d6a67b021e 14153c682b2f 2024-09-13T21:52:56.822 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (9m) 15s ago 19m 509M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:52:56.822 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (8m) 46s ago 17m 443M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:52:56.823 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (7m) 15s ago 19m 59.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:52:56.823 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (7m) 46s ago 17m 45.8M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:52:56.823 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (12m) 15s ago 12m 283M - 4.0 e1d6a67b021e c74feb3f67bf 2024-09-13T21:52:56.823 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (12m) 46s ago 12m 84.8M - 4.0 e1d6a67b021e 63fb12cc3e26 2024-09-13T21:52:56.823 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (9m) 15s ago 18m 15.2M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:52:56.823 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (9m) 46s ago 17m 15.7M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:52:56.824 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (5m) 15s ago 17m 472M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:52:56.824 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (5m) 15s ago 16m 382M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:52:56.824 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (4m) 15s ago 16m 307M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:52:56.824 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (3m) 15s ago 15m 375M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:52:56.824 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (2m) 46s ago 15m 349M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:52:56.825 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (2m) 46s ago 15m 325M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 80819532f2ce 2024-09-13T21:52:56.825 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (103s) 46s ago 14m 308M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 07e99a632a76 2024-09-13T21:52:56.825 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (50s) 46s ago 14m 11.0M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 a55b18452249 2024-09-13T21:52:56.825 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (8m) 15s ago 17m 64.5M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:52:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:56 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:clientreplay 2024-09-13T21:52:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:56 smithi181 ceph-mon[51676]: fsmap foofs:1/1 {0=foofs.smithi154.uynwpr=up:clientreplay} 2024-09-13T21:52:57.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:56 smithi181 ceph-mon[51676]: daemon mds.foofs.smithi154.uynwpr is now active in filesystem foofs as rank 0 2024-09-13T21:52:57.253 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:52:57.254 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:52:57.255 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 13 2024-09-13T21:52:57.255 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:52:57.255 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: "osd", 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: "crash" 2024-09-13T21:52:57.608 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:52:57.609 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "15/23 daemons upgraded", 2024-09-13T21:52:57.609 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading mds daemons", 2024-09-13T21:52:57.609 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:52:57.609 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:52:58.021 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_WARN insufficient standby MDS daemons available 2024-09-13T21:52:58.022 INFO:teuthology.orchestra.run.smithi154.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-13T21:52:58.022 INFO:teuthology.orchestra.run.smithi154.stdout: have 0; want 1 more 2024-09-13T21:52:58.022 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:57 smithi154 ceph-mon[65435]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:58.022 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:57 smithi154 ceph-mon[65435]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:58.022 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:57 smithi154 ceph-mon[65435]: pgmap v334: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.5 MiB/s wr, 157 op/s 2024-09-13T21:52:58.022 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:57 smithi154 ceph-mon[65435]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T21:52:58.022 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:57 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:active 2024-09-13T21:52:58.022 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:57 smithi154 ceph-mon[65435]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 2024-09-13T21:52:58.022 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:57 smithi154 ceph-mon[65435]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:58.022 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:57 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/1788189075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:58.035 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:57 smithi181 ceph-mon[51676]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:58.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:57 smithi181 ceph-mon[51676]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:58.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:57 smithi181 ceph-mon[51676]: pgmap v334: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.5 MiB/s wr, 157 op/s 2024-09-13T21:52:58.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:57 smithi181 ceph-mon[51676]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T21:52:58.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:57 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969] up:active 2024-09-13T21:52:58.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:57 smithi181 ceph-mon[51676]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 2024-09-13T21:52:58.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:57 smithi181 ceph-mon[51676]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:58.037 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:57 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/1788189075' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:52:58.977 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:58 smithi181 ceph-mon[51676]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:58.977 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:58 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2716843253' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:52:59.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:58 smithi154 ceph-mon[65435]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:52:59.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:58 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2716843253' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:52:59.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:59 smithi181 ceph-mon[51676]: pgmap v335: 97 pgs: 97 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.2 MiB/s wr, 90 op/s 2024-09-13T21:52:59.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:59.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:52:59.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:52:59 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:00.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:59 smithi154 ceph-mon[65435]: pgmap v335: 97 pgs: 97 active+clean; 313 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.2 MiB/s wr, 90 op/s 2024-09-13T21:53:00.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:00.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:00.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:52:59 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:01.034 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:00 smithi181 ceph-mon[51676]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T21:53:01.034 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:00 smithi181 ceph-mon[51676]: Cluster is now healthy 2024-09-13T21:53:01.034 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:00 smithi181 ceph-mon[51676]: mds.? [v2:172.21.15.181:6832/2157312466,v1:172.21.15.181:6833/2157312466] up:boot 2024-09-13T21:53:01.034 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:00 smithi181 ceph-mon[51676]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:53:01.034 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:00 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:53:01.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:00 smithi154 ceph-mon[65435]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T21:53:01.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:00 smithi154 ceph-mon[65435]: Cluster is now healthy 2024-09-13T21:53:01.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:00 smithi154 ceph-mon[65435]: mds.? [v2:172.21.15.181:6832/2157312466,v1:172.21.15.181:6833/2157312466] up:boot 2024-09-13T21:53:01.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:00 smithi154 ceph-mon[65435]: fsmap foofs:1 {0=foofs.smithi154.uynwpr=up:active} 1 up:standby 2024-09-13T21:53:01.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:00 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:53:01.992 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:01 smithi181 ceph-mon[51676]: pgmap v336: 97 pgs: 97 active+clean; 259 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.6 MiB/s wr, 81 op/s 2024-09-13T21:53:02.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:01 smithi154 ceph-mon[65435]: pgmap v336: 97 pgs: 97 active+clean; 259 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.6 MiB/s wr, 81 op/s 2024-09-13T21:53:03.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:03.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:03.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:03 smithi154 ceph-mon[65435]: pgmap v337: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.3 MiB/s wr, 74 op/s 2024-09-13T21:53:03.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:03.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:03 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:03.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:03.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:03.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:03 smithi181 ceph-mon[51676]: pgmap v337: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.3 MiB/s wr, 74 op/s 2024-09-13T21:53:03.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:03.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:03 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:05.876 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:05 smithi181 ceph-mon[51676]: pgmap v338: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 57 op/s 2024-09-13T21:53:06.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:05 smithi154 ceph-mon[65435]: pgmap v338: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 57 op/s 2024-09-13T21:53:07.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: Detected new or changed devices on smithi181 2024-09-13T21:53:07.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi154.uynwpr"}]': finished 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi181.mgvpzf"}]': finished 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.091 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:06 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: Detected new or changed devices on smithi181 2024-09-13T21:53:07.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:07.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:07.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi154.uynwpr"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi154.uynwpr"}]': finished 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi181.mgvpzf"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi181.mgvpzf"}]': finished 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:07.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:07.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:06 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:08.164 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all mds 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all rgw 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all iscsi 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: pgmap v339: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 52 op/s 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T21:53:08.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:08.166 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:07 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all mds 2024-09-13T21:53:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all rgw 2024-09-13T21:53:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T21:53:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T21:53:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all iscsi 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: pgmap v339: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 52 op/s 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi154.cmtcgb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:08.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:07 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:09.008 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:08 smithi154 ceph-mon[65435]: Upgrade: Updating nfs.foo.0.0.smithi154.cmtcgb (1/2) 2024-09-13T21:53:09.008 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:08 smithi154 ceph-mon[65435]: Creating key for client.nfs.foo.0.0.smithi154.cmtcgb 2024-09-13T21:53:09.008 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:08 smithi154 ceph-mon[65435]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T21:53:09.008 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:08 smithi154 ceph-mon[65435]: Rados config object exists: conf-nfs.foo 2024-09-13T21:53:09.008 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:08 smithi154 ceph-mon[65435]: Creating key for client.nfs.foo.0.0.smithi154.cmtcgb-rgw 2024-09-13T21:53:09.008 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:08 smithi154 ceph-mon[65435]: Bind address in nfs.foo.0.0.smithi154.cmtcgb's ganesha conf is defaulting to empty 2024-09-13T21:53:09.008 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:08 smithi154 ceph-mon[65435]: Deploying daemon nfs.foo.0.0.smithi154.cmtcgb on smithi154 2024-09-13T21:53:09.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:08 smithi181 ceph-mon[51676]: Upgrade: Updating nfs.foo.0.0.smithi154.cmtcgb (1/2) 2024-09-13T21:53:09.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:08 smithi181 ceph-mon[51676]: Creating key for client.nfs.foo.0.0.smithi154.cmtcgb 2024-09-13T21:53:09.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:08 smithi181 ceph-mon[51676]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T21:53:09.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:08 smithi181 ceph-mon[51676]: Rados config object exists: conf-nfs.foo 2024-09-13T21:53:09.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:08 smithi181 ceph-mon[51676]: Creating key for client.nfs.foo.0.0.smithi154.cmtcgb-rgw 2024-09-13T21:53:09.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:08 smithi181 ceph-mon[51676]: Bind address in nfs.foo.0.0.smithi154.cmtcgb's ganesha conf is defaulting to empty 2024-09-13T21:53:09.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:08 smithi181 ceph-mon[51676]: Deploying daemon nfs.foo.0.0.smithi154.cmtcgb on smithi154 2024-09-13T21:53:10.256 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:09 smithi154 ceph-mon[65435]: pgmap v340: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 355 KiB/s rd, 37 KiB/s wr, 26 op/s 2024-09-13T21:53:10.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:09 smithi181 ceph-mon[51676]: pgmap v340: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 355 KiB/s rd, 37 KiB/s wr, 26 op/s 2024-09-13T21:53:11.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:11.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:53:11.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:11 smithi154 ceph-mon[65435]: pgmap v341: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 37 KiB/s wr, 23 op/s 2024-09-13T21:53:11.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:11 smithi154 ceph-mon[65435]: from='mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969]' entity='mds.foofs.smithi154.uynwpr' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.154:0/4061411931"}]: dispatch 2024-09-13T21:53:11.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:11.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:11 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:53:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:11 smithi181 ceph-mon[51676]: pgmap v341: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 37 KiB/s wr, 23 op/s 2024-09-13T21:53:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:11 smithi181 ceph-mon[51676]: from='mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969]' entity='mds.foofs.smithi154.uynwpr' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.154:0/4061411931"}]: dispatch 2024-09-13T21:53:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:11.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:11 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:12.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:12 smithi181 ceph-mon[51676]: Evicting (and blocklisting) client session 14556 (172.21.15.154:0/4061411931) 2024-09-13T21:53:12.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:12 smithi181 ceph-mon[51676]: from='mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969]' entity='mds.foofs.smithi154.uynwpr' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.154:0/4061411931"}]': finished 2024-09-13T21:53:12.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:12 smithi181 ceph-mon[51676]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T21:53:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:12 smithi154 ceph-mon[65435]: Evicting (and blocklisting) client session 14556 (172.21.15.154:0/4061411931) 2024-09-13T21:53:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:12 smithi154 ceph-mon[65435]: from='mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969]' entity='mds.foofs.smithi154.uynwpr' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.154:0/4061411931"}]': finished 2024-09-13T21:53:12.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:12 smithi154 ceph-mon[65435]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T21:53:13.827 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: pgmap v343: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 2.1 KiB/s wr, 1 op/s 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:13.828 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:13 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: pgmap v343: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 2.1 KiB/s wr, 1 op/s 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.tvxlgt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T21:53:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:13 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:14.736 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:14 smithi181 ceph-mon[51676]: Upgrade: Updating nfs.foo.1.0.smithi181.tvxlgt (2/2) 2024-09-13T21:53:14.737 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:14 smithi181 ceph-mon[51676]: Creating key for client.nfs.foo.1.0.smithi181.tvxlgt 2024-09-13T21:53:14.737 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:14 smithi181 ceph-mon[51676]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T21:53:14.737 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:14 smithi181 ceph-mon[51676]: Rados config object exists: conf-nfs.foo 2024-09-13T21:53:14.737 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:14 smithi181 ceph-mon[51676]: Creating key for client.nfs.foo.1.0.smithi181.tvxlgt-rgw 2024-09-13T21:53:14.737 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:14 smithi181 ceph-mon[51676]: Bind address in nfs.foo.1.0.smithi181.tvxlgt's ganesha conf is defaulting to empty 2024-09-13T21:53:14.737 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:14 smithi181 ceph-mon[51676]: Deploying daemon nfs.foo.1.0.smithi181.tvxlgt on smithi181 2024-09-13T21:53:14.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:14 smithi154 ceph-mon[65435]: Upgrade: Updating nfs.foo.1.0.smithi181.tvxlgt (2/2) 2024-09-13T21:53:14.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:14 smithi154 ceph-mon[65435]: Creating key for client.nfs.foo.1.0.smithi181.tvxlgt 2024-09-13T21:53:14.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:14 smithi154 ceph-mon[65435]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T21:53:14.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:14 smithi154 ceph-mon[65435]: Rados config object exists: conf-nfs.foo 2024-09-13T21:53:14.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:14 smithi154 ceph-mon[65435]: Creating key for client.nfs.foo.1.0.smithi181.tvxlgt-rgw 2024-09-13T21:53:14.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:14 smithi154 ceph-mon[65435]: Bind address in nfs.foo.1.0.smithi181.tvxlgt's ganesha conf is defaulting to empty 2024-09-13T21:53:14.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:14 smithi154 ceph-mon[65435]: Deploying daemon nfs.foo.1.0.smithi181.tvxlgt on smithi181 2024-09-13T21:53:15.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:15 smithi181 ceph-mon[51676]: pgmap v344: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 2.1 KiB/s wr, 1 op/s 2024-09-13T21:53:15.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:15 smithi154 ceph-mon[65435]: pgmap v344: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 2.1 KiB/s wr, 1 op/s 2024-09-13T21:53:17.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:17 smithi181 ceph-mon[51676]: pgmap v345: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T21:53:17.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:17 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:17.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:17 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:17.901 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:17 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:18.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:17 smithi154 ceph-mon[65435]: pgmap v345: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T21:53:18.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:17 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:18.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:17 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:18.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:17 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:19 smithi181 ceph-mon[51676]: pgmap v346: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-13T21:53:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:19 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:19.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:19 smithi181 ceph-mon[51676]: from='mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969]' entity='mds.foofs.smithi154.uynwpr' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.181:0/2693092852"}]: dispatch 2024-09-13T21:53:19.933 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:19 smithi154 ceph-mon[65435]: pgmap v346: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-13T21:53:19.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:19.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:19 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:19.934 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:19 smithi154 ceph-mon[65435]: from='mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969]' entity='mds.foofs.smithi154.uynwpr' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.181:0/2693092852"}]: dispatch 2024-09-13T21:53:21.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:20 smithi181 ceph-mon[51676]: Evicting (and blocklisting) client session 24333 (172.21.15.181:0/2693092852) 2024-09-13T21:53:21.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:20 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:21.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:20 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:21.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:20 smithi181 ceph-mon[51676]: from='mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969]' entity='mds.foofs.smithi154.uynwpr' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.181:0/2693092852"}]': finished 2024-09-13T21:53:21.036 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:20 smithi181 ceph-mon[51676]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T21:53:21.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:20 smithi154 ceph-mon[65435]: Evicting (and blocklisting) client session 24333 (172.21.15.181:0/2693092852) 2024-09-13T21:53:21.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:21.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:20 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:21.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:20 smithi154 ceph-mon[65435]: from='mds.? [v2:172.21.15.154:6834/1269087969,v1:172.21.15.154:6835/1269087969]' entity='mds.foofs.smithi154.uynwpr' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.181:0/2693092852"}]': finished 2024-09-13T21:53:21.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:20 smithi154 ceph-mon[65435]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T21:53:21.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:21 smithi154 ceph-mon[65435]: pgmap v347: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-09-13T21:53:22.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:21 smithi181 ceph-mon[51676]: pgmap v347: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-09-13T21:53:23.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:23.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:23.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:23 smithi154 ceph-mon[65435]: pgmap v349: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-13T21:53:23.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:23.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:23 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:23 smithi181 ceph-mon[51676]: pgmap v349: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-13T21:53:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:23.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:23 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:24.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:24 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:24.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:24 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:24 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:25.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:24 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:26.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:25 smithi154 ceph-mon[65435]: pgmap v350: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-13T21:53:26.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:53:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:25 smithi181 ceph-mon[51676]: pgmap v350: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-13T21:53:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:53:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.589 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.590 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all nfs 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi154.cmtcgb"}]: dispatch 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi154.cmtcgb"}]': finished 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi181.tvxlgt"}]: dispatch 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi181.tvxlgt"}]': finished 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: Upgrade: Setting container_image for all nvmeof 2024-09-13T21:53:27.591 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.592 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.592 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.592 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.592 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:27 smithi154 ceph-mon[65435]: pgmap v351: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-13T21:53:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.650 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all nfs 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi154.cmtcgb"}]: dispatch 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi154.cmtcgb"}]': finished 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi181.tvxlgt"}]: dispatch 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi181.tvxlgt"}]': finished 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: Upgrade: Setting container_image for all nvmeof 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:27.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:27 smithi181 ceph-mon[51676]: pgmap v351: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-13T21:53:28.415 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:53:29.129 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:53:29.129 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (10m) 7s ago 19m 27.2M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:53:29.129 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (7m) 7s ago 19m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:53:29.129 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (7m) 9s ago 18m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:53:29.129 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (10m) 7s ago 18m 53.9M - 8.3.5 dad864ee21e9 435b7350846f 2024-09-13T21:53:29.129 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (55s) 7s ago 13m 29.7M - 19.1.1-324-gecab7be5 0652ff2b5d78 e190f464195a 2024-09-13T21:53:29.129 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (30s) 9s ago 13m 14.5M - 19.1.1-324-gecab7be5 0652ff2b5d78 4e2f9fbe5c81 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (9m) 7s ago 20m 510M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (8m) 9s ago 18m 444M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (8m) 7s ago 20m 62.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (8m) 9s ago 18m 48.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (18s) 7s ago 12m 61.7M - 5.5 0652ff2b5d78 e0f061a4a3bb 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (12s) 9s ago 12m 15.4M - 5.5 0652ff2b5d78 2c2aae7ede7d 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (10m) 7s ago 18m 15.3M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (9m) 9s ago 18m 15.3M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (6m) 7s ago 17m 480M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (5m) 7s ago 17m 386M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (4m) 7s ago 16m 320M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (4m) 7s ago 16m 382M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (3m) 9s ago 15m 352M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:53:29.130 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (2m) 9s ago 15m 364M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 80819532f2ce 2024-09-13T21:53:29.131 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (2m) 9s ago 15m 341M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 07e99a632a76 2024-09-13T21:53:29.131 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (83s) 9s ago 14m 306M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 a55b18452249 2024-09-13T21:53:29.131 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (8m) 7s ago 18m 62.5M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:53:29.539 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:53:29.540 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:53:29.541 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:53:29.541 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:53:29.541 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:53:29.541 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 14 2024-09-13T21:53:29.541 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:53:29.541 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:53:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:29 smithi154 ceph-mon[65435]: Upgrade: Updating grafana.smithi154 2024-09-13T21:53:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:29 smithi154 ceph-mon[65435]: Deploying daemon grafana.smithi154 on smithi154 2024-09-13T21:53:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:29 smithi154 ceph-mon[65435]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:29 smithi154 ceph-mon[65435]: pgmap v352: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-13T21:53:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:29 smithi154 ceph-mon[65435]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:29.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:29 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/2107689552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:29.905 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:53:29.905 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T21:53:29.905 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": true, 2024-09-13T21:53:29.905 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T21:53:29.905 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [ 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: "osd", 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: "mds", 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: "mon", 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr", 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: "crash", 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: "nfs" 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": "18/23 daemons upgraded", 2024-09-13T21:53:29.906 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "Currently upgrading grafana daemons", 2024-09-13T21:53:29.907 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:53:29.907 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:53:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:29 smithi181 ceph-mon[51676]: Upgrade: Updating grafana.smithi154 2024-09-13T21:53:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:29 smithi181 ceph-mon[51676]: Deploying daemon grafana.smithi154 on smithi154 2024-09-13T21:53:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:29 smithi181 ceph-mon[51676]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:29 smithi181 ceph-mon[51676]: pgmap v352: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-13T21:53:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:29 smithi181 ceph-mon[51676]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:29 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/2107689552' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:30.315 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:53:31.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:30 smithi154 ceph-mon[65435]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:31.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:30 smithi154 ceph-mon[65435]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:31.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:30 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/248955948' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:53:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:30 smithi181 ceph-mon[51676]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:30 smithi181 ceph-mon[51676]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:53:31.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:30 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/248955948' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:53:31.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:31 smithi154 ceph-mon[65435]: pgmap v353: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 307 B/s wr, 0 op/s 2024-09-13T21:53:32.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:31 smithi181 ceph-mon[51676]: pgmap v353: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 307 B/s wr, 0 op/s 2024-09-13T21:53:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:33 smithi154 ceph-mon[65435]: pgmap v354: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 256 B/s wr, 0 op/s 2024-09-13T21:53:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:33 smithi181 ceph-mon[51676]: pgmap v354: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 256 B/s wr, 0 op/s 2024-09-13T21:53:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:35 smithi154 ceph-mon[65435]: pgmap v355: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T21:53:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:35 smithi181 ceph-mon[51676]: pgmap v355: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T21:53:37.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:37 smithi154 ceph-mon[65435]: pgmap v356: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T21:53:37.952 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:37 smithi181 ceph-mon[51676]: pgmap v356: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T21:53:39.963 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:39 smithi154 ceph-mon[65435]: pgmap v357: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:39 smithi181 ceph-mon[51676]: pgmap v357: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:40.961 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:40 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:53:41.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:40 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:53:42.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:41 smithi181 ceph-mon[51676]: pgmap v358: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:53:42.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:42.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:42.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:41 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:42.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:41 smithi154 ceph-mon[65435]: pgmap v358: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:53:42.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:42.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:42.335 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:41 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:43 smithi181 ceph-mon[51676]: pgmap v359: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:44.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:43 smithi154 ceph-mon[65435]: pgmap v359: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:45 smithi181 ceph-mon[51676]: pgmap v360: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:46.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:45 smithi154 ceph-mon[65435]: pgmap v360: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:47.269 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:47.269 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:47.270 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:47.270 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:46 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:47.270 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:46 smithi154 ceph-mon[65435]: pgmap v361: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:53:47.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:47.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:47.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:47.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:46 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:47.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:46 smithi181 ceph-mon[51676]: pgmap v361: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:53:49.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:49 smithi154 ceph-mon[65435]: pgmap v362: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:50.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:49 smithi181 ceph-mon[51676]: pgmap v362: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:51.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.340 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.341 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T21:53:51.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:51.344 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.345 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.345 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.401 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T21:53:51.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T21:53:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.404 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:53:51.405 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:53:51.406 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.406 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:51.406 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:53:52.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:51 smithi154 ceph-mon[65435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:52.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:51 smithi154 ceph-mon[65435]: Upgrade: Finalizing container_image settings 2024-09-13T21:53:52.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:51 smithi154 ceph-mon[65435]: Upgrade: Complete! 2024-09-13T21:53:52.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:51 smithi154 ceph-mon[65435]: pgmap v363: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:53:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:51 smithi181 ceph-mon[51676]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T21:53:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:51 smithi181 ceph-mon[51676]: Upgrade: Finalizing container_image settings 2024-09-13T21:53:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:51 smithi181 ceph-mon[51676]: Upgrade: Complete! 2024-09-13T21:53:52.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:51 smithi181 ceph-mon[51676]: pgmap v363: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:53:54.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:53 smithi154 ceph-mon[65435]: pgmap v364: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:54.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:53 smithi181 ceph-mon[51676]: pgmap v364: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:56.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:55 smithi154 ceph-mon[65435]: pgmap v365: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:56.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:53:56.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:55 smithi181 ceph-mon[51676]: pgmap v365: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:53:56.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:53:58.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:57 smithi154 ceph-mon[65435]: pgmap v366: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:53:58.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:57 smithi181 ceph-mon[51676]: pgmap v366: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:00.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:53:59 smithi154 ceph-mon[65435]: pgmap v367: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:00.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:53:59 smithi181 ceph-mon[51676]: pgmap v367: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:01.798 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T21:54:02.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:01 smithi154 ceph-mon[65435]: pgmap v368: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:02.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:01 smithi154 ceph-mon[65435]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:54:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:01 smithi181 ceph-mon[51676]: pgmap v368: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:02.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:01 smithi181 ceph-mon[51676]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:54:03.815 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (11m) 17s ago 19m 26.8M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (7m) 17s ago 19m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (7m) 44s ago 19m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (23s) 17s ago 19m 28.9M - 9.4.12 2bacad6d85d8 ab7b93dd5c90 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (90s) 17s ago 13m 29.3M - 19.1.1-324-gecab7be5 0652ff2b5d78 e190f464195a 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (64s) 44s ago 13m 14.5M - 19.1.1-324-gecab7be5 0652ff2b5d78 4e2f9fbe5c81 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (10m) 17s ago 20m 518M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (9m) 44s ago 19m 444M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (9m) 17s ago 21m 62.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (8m) 44s ago 18m 48.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (52s) 17s ago 13m 62.2M - 5.5 0652ff2b5d78 e0f061a4a3bb 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (47s) 44s ago 13m 15.4M - 5.5 0652ff2b5d78 2c2aae7ede7d 2024-09-13T21:54:03.816 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (10m) 17s ago 19m 15.1M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (10m) 44s ago 18m 15.3M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (7m) 17s ago 18m 481M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (6m) 17s ago 17m 386M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (5m) 17s ago 17m 320M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (4m) 17s ago 16m 381M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (4m) 44s ago 16m 352M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (3m) 44s ago 16m 364M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 80819532f2ce 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (2m) 44s ago 15m 341M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 07e99a632a76 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (117s) 44s ago 15m 306M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 a55b18452249 2024-09-13T21:54:03.817 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (9m) 17s ago 18m 64.5M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:54:04.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:03 smithi154 ceph-mon[65435]: pgmap v369: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:04.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:03 smithi181 ceph-mon[51676]: pgmap v369: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:04.521 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T21:54:05.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:04 smithi154 ceph-mon[65435]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:54:05.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:04 smithi181 ceph-mon[51676]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:54:06.240 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:05 smithi154 ceph-mon[65435]: pgmap v370: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:06.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:05 smithi181 ceph-mon[51676]: pgmap v370: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:06.753 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:54:06.753 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 14 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:54:06.754 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:54:07.071 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:06 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/3252232958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:54:07.301 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:06 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/3252232958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:54:07.444 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-13T21:54:08.016 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:07 smithi154 ceph-mon[65435]: pgmap v371: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:08.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:07 smithi181 ceph-mon[51676]: pgmap v371: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:09.560 INFO:teuthology.orchestra.run.smithi154.stdout:wait for servicemap items w/ changing names to refresh 2024-09-13T21:54:10.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:09 smithi154 ceph-mon[65435]: pgmap v372: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:10.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:09 smithi181 ceph-mon[51676]: pgmap v372: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:10.420 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 60' 2024-09-13T21:54:11.031 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:54:11.031 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:10 smithi154 ceph-mon[65435]: pgmap v373: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:54:11.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:10 smithi181 ceph-mon[51676]: pgmap v373: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:14.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:13 smithi154 ceph-mon[65435]: pgmap v374: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:13 smithi181 ceph-mon[51676]: pgmap v374: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:16.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:15 smithi154 ceph-mon[65435]: pgmap v375: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:15 smithi181 ceph-mon[51676]: pgmap v375: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:18.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:17 smithi154 ceph-mon[65435]: pgmap v376: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:18.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:17 smithi181 ceph-mon[51676]: pgmap v376: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:20.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:19 smithi154 ceph-mon[65435]: pgmap v377: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:20.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:19 smithi181 ceph-mon[51676]: pgmap v377: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:21.946 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:21 smithi154 ceph-mon[65435]: pgmap v378: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:21 smithi181 ceph-mon[51676]: pgmap v378: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:24.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:23 smithi154 ceph-mon[65435]: pgmap v379: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:23 smithi181 ceph-mon[51676]: pgmap v379: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:26.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:25 smithi154 ceph-mon[65435]: pgmap v380: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:26.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:54:26.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:25 smithi181 ceph-mon[51676]: pgmap v380: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:26.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:54:28.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:27 smithi154 ceph-mon[65435]: pgmap v381: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:27 smithi181 ceph-mon[51676]: pgmap v381: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:30.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:29 smithi154 ceph-mon[65435]: pgmap v382: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:29 smithi181 ceph-mon[51676]: pgmap v382: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:32.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:31 smithi154 ceph-mon[65435]: pgmap v383: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:32.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:31 smithi181 ceph-mon[51676]: pgmap v383: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:34.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:33 smithi154 ceph-mon[65435]: pgmap v384: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:33 smithi181 ceph-mon[51676]: pgmap v384: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:35 smithi154 ceph-mon[65435]: pgmap v385: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:35 smithi181 ceph-mon[51676]: pgmap v385: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:38.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:37 smithi154 ceph-mon[65435]: pgmap v386: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:37 smithi181 ceph-mon[51676]: pgmap v386: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:40.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:39 smithi154 ceph-mon[65435]: pgmap v387: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:39 smithi181 ceph-mon[51676]: pgmap v387: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:41.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:40 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:54:41.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:40 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:54:42.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:41 smithi154 ceph-mon[65435]: pgmap v388: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:42.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:41 smithi181 ceph-mon[51676]: pgmap v388: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:44.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:43 smithi154 ceph-mon[65435]: pgmap v389: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:44.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:43 smithi181 ceph-mon[51676]: pgmap v389: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:46.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:45 smithi154 ceph-mon[65435]: pgmap v390: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:46.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:45 smithi181 ceph-mon[51676]: pgmap v390: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:48.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:47 smithi154 ceph-mon[65435]: pgmap v391: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:48.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:47 smithi181 ceph-mon[51676]: pgmap v391: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:50.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:49 smithi154 ceph-mon[65435]: pgmap v392: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:50.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:49 smithi181 ceph-mon[51676]: pgmap v392: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:51.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:54:51.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:54:51.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:50 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:54:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T21:54:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:54:51.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:50 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:54:52.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:51 smithi154 ceph-mon[65435]: pgmap v393: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:52.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:54:52.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:54:52.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:54:52.090 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:51 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:54:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:51 smithi181 ceph-mon[51676]: pgmap v393: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T21:54:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T21:54:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:54:52.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:51 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' 2024-09-13T21:54:54.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:53 smithi154 ceph-mon[65435]: pgmap v394: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:54.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:53 smithi181 ceph-mon[51676]: pgmap v394: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:56.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:55 smithi154 ceph-mon[65435]: pgmap v395: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:56.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:55 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:54:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:55 smithi181 ceph-mon[51676]: pgmap v395: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:54:56.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:55 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:54:58.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:57 smithi154 ceph-mon[65435]: pgmap v396: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:54:58.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:57 smithi181 ceph-mon[51676]: pgmap v396: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:00.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:54:59 smithi154 ceph-mon[65435]: pgmap v397: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:00.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:54:59 smithi181 ceph-mon[51676]: pgmap v397: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:02.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:01 smithi154 ceph-mon[65435]: pgmap v398: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:02.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:01 smithi181 ceph-mon[51676]: pgmap v398: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:04.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:03 smithi154 ceph-mon[65435]: pgmap v399: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:04.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:03 smithi181 ceph-mon[51676]: pgmap v399: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:06.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:05 smithi154 ceph-mon[65435]: pgmap v400: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:06.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:05 smithi181 ceph-mon[51676]: pgmap v400: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:08.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:07 smithi154 ceph-mon[65435]: pgmap v401: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:08.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:07 smithi181 ceph-mon[51676]: pgmap v401: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:10.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:09 smithi154 ceph-mon[65435]: pgmap v402: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:10.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:09 smithi181 ceph-mon[51676]: pgmap v402: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:11.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:10 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:55:11.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:10 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:55:12.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:11 smithi154 ceph-mon[65435]: pgmap v403: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:12.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:11 smithi181 ceph-mon[51676]: pgmap v403: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:12.934 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T21:55:13.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:13 smithi154 ceph-mon[65435]: pgmap v404: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:14.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:13 smithi181 ceph-mon[51676]: pgmap v404: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:14.913 INFO:teuthology.orchestra.run.smithi154.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T21:55:14.913 INFO:teuthology.orchestra.run.smithi154.stdout:alertmanager.smithi154 smithi154 *:9093,9094 running (12m) 88s ago 20m 26.8M - 0.25.0 c8568f914cd2 fe6d8c987157 2024-09-13T21:55:14.913 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi154 smithi154 running (8m) 88s ago 20m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 29dd0668152f 2024-09-13T21:55:14.913 INFO:teuthology.orchestra.run.smithi154.stdout:crash.smithi181 smithi181 running (8m) 115s ago 20m 6899k - 19.1.1-324-gecab7be5 0652ff2b5d78 9f55f235844e 2024-09-13T21:55:14.913 INFO:teuthology.orchestra.run.smithi154.stdout:grafana.smithi154 smithi154 *:3000 running (94s) 88s ago 20m 28.9M - 9.4.12 2bacad6d85d8 ab7b93dd5c90 2024-09-13T21:55:14.913 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi154.uynwpr smithi154 running (2m) 88s ago 15m 29.3M - 19.1.1-324-gecab7be5 0652ff2b5d78 e190f464195a 2024-09-13T21:55:14.913 INFO:teuthology.orchestra.run.smithi154.stdout:mds.foofs.smithi181.mgvpzf smithi181 running (2m) 115s ago 14m 14.5M - 19.1.1-324-gecab7be5 0652ff2b5d78 4e2f9fbe5c81 2024-09-13T21:55:14.913 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi154.ayrwdi smithi154 *:8443,9283,8765 running (11m) 88s ago 22m 518M - 19.1.1-324-gecab7be5 0652ff2b5d78 beba2bc4538a 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:mgr.smithi181.byvkib smithi181 *:8443,9283,8765 running (10m) 115s ago 20m 444M - 19.1.1-324-gecab7be5 0652ff2b5d78 94949cbf5a10 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi154 smithi154 running (10m) 88s ago 22m 62.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 44ff9ab2ade6 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:mon.smithi181 smithi181 running (9m) 115s ago 20m 48.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 cf973304eaff 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.0.0.smithi154.cmtcgb smithi154 *:2049 running (2m) 88s ago 14m 62.2M - 5.5 0652ff2b5d78 e0f061a4a3bb 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:nfs.foo.1.0.smithi181.tvxlgt smithi181 *:2049 running (118s) 115s ago 14m 15.4M - 5.5 0652ff2b5d78 2c2aae7ede7d 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi154 smithi154 *:9100 running (12m) 88s ago 20m 15.1M - 1.5.0 0da6a335fe13 fa81e3dabb09 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:node-exporter.smithi181 smithi181 *:9100 running (11m) 115s ago 20m 15.3M - 1.5.0 0da6a335fe13 9b644a10ce37 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:osd.0 smithi154 running (8m) 88s ago 19m 481M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 6e650440345f 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:osd.1 smithi154 running (7m) 88s ago 18m 386M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 91c024a3fc2e 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:osd.2 smithi154 running (6m) 88s ago 18m 320M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 cc7d439fad81 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:osd.3 smithi154 running (5m) 88s ago 18m 381M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 287dcaef58cf 2024-09-13T21:55:14.914 INFO:teuthology.orchestra.run.smithi154.stdout:osd.4 smithi181 running (5m) 115s ago 17m 352M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 e8ce19a28c45 2024-09-13T21:55:14.915 INFO:teuthology.orchestra.run.smithi154.stdout:osd.5 smithi181 running (4m) 115s ago 17m 364M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 80819532f2ce 2024-09-13T21:55:14.915 INFO:teuthology.orchestra.run.smithi154.stdout:osd.6 smithi181 running (4m) 115s ago 16m 341M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 07e99a632a76 2024-09-13T21:55:14.915 INFO:teuthology.orchestra.run.smithi154.stdout:osd.7 smithi181 running (3m) 115s ago 16m 306M 2703M 19.1.1-324-gecab7be5 0652ff2b5d78 a55b18452249 2024-09-13T21:55:14.915 INFO:teuthology.orchestra.run.smithi154.stdout:prometheus.smithi154 smithi154 *:9095 running (10m) 88s ago 19m 64.5M - 2.43.0 a07b618ecd1d 247df0f8efb5 2024-09-13T21:55:15.549 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T21:55:16.087 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:15 smithi154 ceph-mon[65435]: pgmap v405: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:16.088 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:15 smithi154 ceph-mon[65435]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:55:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:15 smithi181 ceph-mon[51676]: pgmap v405: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:16.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:15 smithi181 ceph-mon[51676]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:55:17.559 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:55:17.560 INFO:teuthology.orchestra.run.smithi154.stdout: "target_image": null, 2024-09-13T21:55:17.560 INFO:teuthology.orchestra.run.smithi154.stdout: "in_progress": false, 2024-09-13T21:55:17.560 INFO:teuthology.orchestra.run.smithi154.stdout: "which": "", 2024-09-13T21:55:17.560 INFO:teuthology.orchestra.run.smithi154.stdout: "services_complete": [], 2024-09-13T21:55:17.560 INFO:teuthology.orchestra.run.smithi154.stdout: "progress": null, 2024-09-13T21:55:17.560 INFO:teuthology.orchestra.run.smithi154.stdout: "message": "", 2024-09-13T21:55:17.560 INFO:teuthology.orchestra.run.smithi154.stdout: "is_paused": false 2024-09-13T21:55:17.560 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:55:17.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:17 smithi154 ceph-mon[65435]: pgmap v406: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:18.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:17 smithi181 ceph-mon[51676]: pgmap v406: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:18.154 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T21:55:18.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:18 smithi181 ceph-mon[51676]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:55:19.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:18 smithi154 ceph-mon[65435]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:55:20.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:19 smithi154 ceph-mon[65435]: pgmap v407: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:20.120 INFO:teuthology.orchestra.run.smithi154.stdout:HEALTH_OK 2024-09-13T21:55:20.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:19 smithi181 ceph-mon[51676]: pgmap v407: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:20.704 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T21:55:21.009 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:20 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/1499235301' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:55:21.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:20 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/1499235301' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T21:55:21.995 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:21 smithi154 ceph-mon[65435]: pgmap v408: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:22.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:21 smithi181 ceph-mon[51676]: pgmap v408: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:22.752 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-09-13T21:55:22.781 INFO:teuthology.orchestra.run.smithi154.stdout: "mon": { 2024-09-13T21:55:22.781 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: "mgr": { 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: "osd": { 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: "mds": { 2024-09-13T21:55:22.782 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T21:55:22.783 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:55:22.783 INFO:teuthology.orchestra.run.smithi154.stdout: "overall": { 2024-09-13T21:55:22.783 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 14 2024-09-13T21:55:22.783 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:55:22.783 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-09-13T21:55:23.053 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:22 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/1634884164' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:55:23.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:22 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/1634884164' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:55:23.343 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-13T21:55:23.932 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:23 smithi154 ceph-mon[65435]: pgmap v409: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:24.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:23 smithi181 ceph-mon[51676]: pgmap v409: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:25.442 INFO:teuthology.orchestra.run.smithi154.stdout:true 2024-09-13T21:55:26.020 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:25 smithi154 ceph-mon[65435]: pgmap v410: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:26.020 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:25 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/3577001272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:55:26.020 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:25 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:55:26.043 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-13T21:55:26.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:25 smithi181 ceph-mon[51676]: pgmap v410: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:25 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/3577001272' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:55:26.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:25 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:55:28.011 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T21:55:28.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:27 smithi154 ceph-mon[65435]: pgmap v411: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:28.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:27 smithi181 ceph-mon[51676]: pgmap v411: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:28.566 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-13T21:55:28.567 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-13T21:55:28.567 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T21:55:29.078 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:28 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/357590791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:55:29.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:28 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/357590791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T21:55:30.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:29 smithi154 ceph-mon[65435]: pgmap v412: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:30.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:29 smithi181 ceph-mon[51676]: pgmap v412: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:30.525 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-09-13T21:55:30.525 INFO:teuthology.orchestra.run.smithi154.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T21:33:45.485648Z", "last_refresh": "2024-09-13T21:53:45.972717Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T21:33:40.086868Z", "last_refresh": "2024-09-13T21:53:19.641711Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-13T21:33:43.093300Z", "last_refresh": "2024-09-13T21:53:45.972872Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T21:40:09.720613Z", "last_refresh": "2024-09-13T21:53:19.642858Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T21:33:38.169388Z", "last_refresh": "2024-09-13T21:53:19.641876Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi154:172.21.15.154=smithi154", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T21:34:27.330740Z", "last_refresh": "2024-09-13T21:53:19.642020Z", "running": 2, "size": 2}}, {"events": ["2024-09-13T21:54:51.035072Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-13T21:40:30.782981Z", "last_refresh": "2024-09-13T21:53:19.642999Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T21:33:44.269558Z", "last_refresh": "2024-09-13T21:53:19.642162Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", "last_refresh": "2024-09-13T21:53:19.642303Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T21:33:41.241307Z", "last_refresh": "2024-09-13T21:53:45.973013Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T21:55:31.188 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-13T21:55:31.189 DEBUG:teuthology.parallel:result is None 2024-09-13T21:55:31.189 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T21:55:31.199 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:55:31.199 DEBUG:teuthology.orchestra.run.smithi154:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-13T21:55:31.225 INFO:teuthology.orchestra.run.smithi154.stderr:+ umount /mnt/foo 2024-09-13T21:55:31.256 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T21:55:31.265 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:55:31.265 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-13T21:55:31.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:31 smithi154 ceph-mon[65435]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:55:31.839 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:31 smithi154 ceph-mon[65435]: pgmap v413: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:32.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:31 smithi181 ceph-mon[51676]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T21:55:32.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:31 smithi181 ceph-mon[51676]: pgmap v413: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:33.332 INFO:teuthology.orchestra.run.smithi154.stdout: "foo" 2024-09-13T21:55:33.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:33 smithi154 ceph-mon[65435]: pgmap v414: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:33.971 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-13T21:55:34.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:33 smithi181 ceph-mon[51676]: pgmap v414: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:35.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:34 smithi154 ceph-mon[65435]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:55:35.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:34 smithi181 ceph-mon[51676]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:55:36.034 INFO:teuthology.orchestra.run.smithi154.stdout:[ 2024-09-13T21:55:36.034 INFO:teuthology.orchestra.run.smithi154.stdout: { 2024-09-13T21:55:36.034 INFO:teuthology.orchestra.run.smithi154.stdout: "access_type": "RW", 2024-09-13T21:55:36.034 INFO:teuthology.orchestra.run.smithi154.stdout: "clients": [], 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "cluster_id": "foo", 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "export_id": 1, 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "fsal": { 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "fs_name": "foofs", 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "name": "CEPH", 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "user_id": "nfs.foo.1" 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: }, 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "path": "/", 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "protocols": [ 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: 4 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: ], 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "pseudo": "/fake", 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "security_label": true, 2024-09-13T21:55:36.035 INFO:teuthology.orchestra.run.smithi154.stdout: "squash": "none", 2024-09-13T21:55:36.036 INFO:teuthology.orchestra.run.smithi154.stdout: "transports": [ 2024-09-13T21:55:36.036 INFO:teuthology.orchestra.run.smithi154.stdout: "TCP" 2024-09-13T21:55:36.036 INFO:teuthology.orchestra.run.smithi154.stdout: ] 2024-09-13T21:55:36.036 INFO:teuthology.orchestra.run.smithi154.stdout: } 2024-09-13T21:55:36.036 INFO:teuthology.orchestra.run.smithi154.stdout:] 2024-09-13T21:55:36.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:35 smithi154 ceph-mon[65435]: pgmap v415: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:36.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:35 smithi181 ceph-mon[51676]: pgmap v415: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:36.635 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-13T21:55:37.861 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:37 smithi154 ceph-mon[65435]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:55:37.862 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:37 smithi154 ceph-mon[65435]: pgmap v416: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:37.862 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:37 smithi154 ceph-mon[65435]: mgrmap e44: smithi154.ayrwdi(active, since 10m), standbys: smithi181.byvkib 2024-09-13T21:55:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:37 smithi181 ceph-mon[51676]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T21:55:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:37 smithi181 ceph-mon[51676]: pgmap v416: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T21:55:38.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:37 smithi181 ceph-mon[51676]: mgrmap e44: smithi154.ayrwdi(active, since 10m), standbys: smithi181.byvkib 2024-09-13T21:55:38.306 INFO:teuthology.orchestra.run.smithi154.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-13T21:55:38.306 INFO:teuthology.orchestra.run.smithi154.stdout:foo grace 2024-09-13T21:55:38.306 INFO:teuthology.orchestra.run.smithi154.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-13T21:55:38.306 INFO:teuthology.orchestra.run.smithi154.stdout:foo conf-nfs.foo 2024-09-13T21:55:38.307 INFO:teuthology.orchestra.run.smithi154.stdout:foo export-1 2024-09-13T21:55:38.921 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T21:55:38.930 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T21:55:38.930 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ad78d70a-7217-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-13T21:55:38.930 DEBUG:teuthology.orchestra.run.smithi154:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-13T21:55:38.930 DEBUG:teuthology.orchestra.run.smithi154:> ' 2024-09-13T21:55:40.089 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:39 smithi154 ceph-mon[65435]: pgmap v417: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:40.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:39 smithi181 ceph-mon[51676]: pgmap v417: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T21:55:40.764 INFO:teuthology.orchestra.run.smithi154.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-13T21:55:41.002 INFO:teuthology.orchestra.run.smithi154.stderr:+ [[ 7 -gt 2 ]] 2024-09-13T21:55:41.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:40 smithi181 ceph-mon[51676]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:55:41.310 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:40 smithi154 ceph-mon[65435]: from='mgr.34104 172.21.15.154:0/4051027222' entity='mgr.smithi154.ayrwdi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T21:55:41.716 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T21:55:41.727 INFO:tasks.cephadm:Teardown begin 2024-09-13T21:55:41.728 DEBUG:teuthology.orchestra.run.smithi154:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T21:55:41.755 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T21:55:41.787 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T21:55:41.787 DEBUG:teuthology.orchestra.run.smithi154:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T21:55:41.810 DEBUG:teuthology.orchestra.run.smithi181:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T21:55:41.844 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T21:55:41.845 INFO:tasks.cephadm.mon.smithi154:Stopping mon.smithi154... 2024-09-13T21:55:41.845 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154 2024-09-13T21:55:41.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:41 smithi154 ceph-mon[65435]: pgmap v418: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T21:55:41.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:41 smithi154 ceph-mon[65435]: from='client.? 172.21.15.154:0/995257252' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-13T21:55:41.947 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:41 smithi154 systemd[1]: Stopping Ceph mon.smithi154 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:55:42.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:41 smithi181 ceph-mon[51676]: pgmap v418: 97 pgs: 97 active+clean; 243 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T21:55:42.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:41 smithi181 ceph-mon[51676]: from='client.? 172.21.15.154:0/995257252' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-13T21:55:42.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:42 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[65431]: 2024-09-13T21:55:42.170+0000 7f7c1f934640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi154 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:55:42.339 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:42 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154[65431]: 2024-09-13T21:55:42.170+0000 7f7c1f934640 -1 mon.smithi154@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T21:55:42.674 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:42 smithi154 podman[98560]: 2024-09-13 21:55:42.396402805 +0000 UTC m=+0.348728997 container died 44ff9ab2ade69caf91969ce179e435d14bc4979a1c852441dd4c25690c9dee5a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T21:55:43.057 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:42 smithi154 podman[98560]: 2024-09-13 21:55:42.771662572 +0000 UTC m=+0.723988761 container cleanup 44ff9ab2ade69caf91969ce179e435d14bc4979a1c852441dd4c25690c9dee5a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T21:55:43.057 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Sep 13 21:55:42 smithi154 bash[98560]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi154 2024-09-13T21:55:43.198 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi154.service' 2024-09-13T21:55:43.246 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:55:43.246 INFO:tasks.cephadm.mon.smithi154:Stopped mon.smithi154 2024-09-13T21:55:43.246 INFO:tasks.cephadm.mon.smithi181:Stopping mon.smithi181... 2024-09-13T21:55:43.246 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi181 2024-09-13T21:55:43.564 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:43 smithi181 systemd[1]: Stopping Ceph mon.smithi181 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:55:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:43 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181[51672]: 2024-09-13T21:55:43.562+0000 7fa2bd71e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi181 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:55:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:43 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181[51672]: 2024-09-13T21:55:43.562+0000 7fa2bd71e640 -1 mon.smithi181@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T21:55:43.900 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:43 smithi181 podman[73205]: 2024-09-13 21:55:43.746090446 +0000 UTC m=+0.296740541 container died cf973304eaffa760c4aee5b19c947336ddf76ef6bdd7c42f0b9c21775adcd9f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:55:44.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:44 smithi181 podman[73205]: 2024-09-13 21:55:44.19617189 +0000 UTC m=+0.746822018 container cleanup cf973304eaffa760c4aee5b19c947336ddf76ef6bdd7c42f0b9c21775adcd9f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T21:55:44.400 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Sep 13 21:55:44 smithi181 bash[73205]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-mon-smithi181 2024-09-13T21:55:44.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@mon.smithi181.service' 2024-09-13T21:55:44.659 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:55:44.659 INFO:tasks.cephadm.mon.smithi181:Stopped mon.smithi181 2024-09-13T21:55:44.660 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T21:55:44.660 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.0 2024-09-13T21:55:44.969 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:44 smithi154 systemd[1]: Stopping Ceph osd.0 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:55:45.339 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:44 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[71454]: 2024-09-13T21:55:44.966+0000 7f1dab011640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:55:45.339 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:44 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[71454]: 2024-09-13T21:55:44.966+0000 7f1dab011640 -1 osd.0 172 *** Got signal Terminated *** 2024-09-13T21:55:45.339 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:44 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0[71454]: 2024-09-13T21:55:44.966+0000 7f1dab011640 -1 osd.0 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:55:50.770 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:50 smithi154 podman[98723]: 2024-09-13 21:55:50.48946498 +0000 UTC m=+5.630670800 container died 6e650440345fd0fe0359c1aa262f74277108550b523d39c47ab1a3e2a3be10f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T21:55:51.089 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:50 smithi154 podman[98723]: 2024-09-13 21:55:50.851452579 +0000 UTC m=+5.992658402 container cleanup 6e650440345fd0fe0359c1aa262f74277108550b523d39c47ab1a3e2a3be10f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:55:51.089 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:50 smithi154 bash[98723]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0 2024-09-13T21:55:51.466 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:51 smithi154 podman[98740]: 2024-09-13 21:55:51.136523423 +0000 UTC m=+0.643695923 container remove 6e650440345fd0fe0359c1aa262f74277108550b523d39c47ab1a3e2a3be10f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T21:55:51.783 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:51 smithi154 podman[98827]: 2024-09-13 21:55:51.368438316 +0000 UTC m=+0.016887074 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:55:51.783 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:51 smithi154 podman[98827]: 2024-09-13 21:55:51.504096562 +0000 UTC m=+0.152545310 container create 076a5c296833c8e0ea746de96a5a41638c300b2491203fe16c1b8dce731a89a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:55:51.783 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:51 smithi154 podman[98827]: 2024-09-13 21:55:51.706747879 +0000 UTC m=+0.355196624 container init 076a5c296833c8e0ea746de96a5a41638c300b2491203fe16c1b8dce731a89a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T21:55:51.783 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:51 smithi154 podman[98827]: 2024-09-13 21:55:51.710031762 +0000 UTC m=+0.358480518 container start 076a5c296833c8e0ea746de96a5a41638c300b2491203fe16c1b8dce731a89a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T21:55:51.783 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:51 smithi154 podman[98827]: 2024-09-13 21:55:51.781807643 +0000 UTC m=+0.430256393 container attach 076a5c296833c8e0ea746de96a5a41638c300b2491203fe16c1b8dce731a89a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:55:52.089 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:51 smithi154 podman[98827]: 2024-09-13 21:55:51.897873341 +0000 UTC m=+0.546322095 container died 076a5c296833c8e0ea746de96a5a41638c300b2491203fe16c1b8dce731a89a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T21:55:52.448 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:52 smithi154 podman[98850]: 2024-09-13 21:55:52.4188466 +0000 UTC m=+0.515408649 container remove 076a5c296833c8e0ea746de96a5a41638c300b2491203fe16c1b8dce731a89a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be) 2024-09-13T21:55:52.448 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:52 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T21:55:52.448 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:52 smithi154 systemd[1]: Stopped Ceph osd.0 for ad78d70a-7217-11ef-bceb-c7b262605968. 2024-09-13T21:55:52.448 INFO:journalctl@ceph.osd.0.smithi154.stdout:Sep 13 21:55:52 smithi154 systemd[1]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 45.473s CPU time. 2024-09-13T21:55:52.449 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T21:55:52.487 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:55:52.487 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T21:55:52.487 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T21:55:52.488 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.1 2024-09-13T21:55:52.839 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:52 smithi154 systemd[1]: Stopping Ceph osd.1 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:55:53.339 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:52 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[74076]: 2024-09-13T21:55:52.838+0000 7fe264954640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:55:53.406 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:52 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[74076]: 2024-09-13T21:55:52.838+0000 7fe264954640 -1 osd.1 172 *** Got signal Terminated *** 2024-09-13T21:55:53.406 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:52 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1[74076]: 2024-09-13T21:55:52.838+0000 7fe264954640 -1 osd.1 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:55:58.707 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:58 smithi154 podman[98930]: 2024-09-13 21:55:58.412623266 +0000 UTC m=+5.694397407 container died 91c024a3fc2e74b8b9b05e36faeece13a7feea0525b43383dfc939f304c2a7df (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:55:59.089 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:58 smithi154 podman[98930]: 2024-09-13 21:55:58.796983994 +0000 UTC m=+6.078758136 container cleanup 91c024a3fc2e74b8b9b05e36faeece13a7feea0525b43383dfc939f304c2a7df (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-13T21:55:59.089 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:58 smithi154 bash[98930]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1 2024-09-13T21:55:59.089 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:58 smithi154 podman[98949]: 2024-09-13 21:55:58.956591632 +0000 UTC m=+0.535593998 container remove 91c024a3fc2e74b8b9b05e36faeece13a7feea0525b43383dfc939f304c2a7df (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T21:55:59.694 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:59 smithi154 podman[99038]: 2024-09-13 21:55:59.275242002 +0000 UTC m=+0.016234984 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:55:59.694 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:59 smithi154 podman[99038]: 2024-09-13 21:55:59.398959809 +0000 UTC m=+0.139952780 container create ea9621c4b7ec1d17c537cb9ae5711039cc5d639bd1a9452c0c32c7d550e33522 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T21:55:59.694 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:59 smithi154 podman[99038]: 2024-09-13 21:55:59.601538542 +0000 UTC m=+0.342531519 container init ea9621c4b7ec1d17c537cb9ae5711039cc5d639bd1a9452c0c32c7d550e33522 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:55:59.694 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:59 smithi154 podman[99038]: 2024-09-13 21:55:59.605313568 +0000 UTC m=+0.346306538 container start ea9621c4b7ec1d17c537cb9ae5711039cc5d639bd1a9452c0c32c7d550e33522 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T21:55:59.694 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:59 smithi154 podman[99038]: 2024-09-13 21:55:59.693156665 +0000 UTC m=+0.434149634 container attach ea9621c4b7ec1d17c537cb9ae5711039cc5d639bd1a9452c0c32c7d550e33522 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:56:00.081 INFO:journalctl@ceph.osd.1.smithi154.stdout:Sep 13 21:55:59 smithi154 podman[99038]: 2024-09-13 21:55:59.793646047 +0000 UTC m=+0.534639034 container died ea9621c4b7ec1d17c537cb9ae5711039cc5d639bd1a9452c0c32c7d550e33522 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:56:00.354 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T21:56:00.393 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:56:00.393 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T21:56:00.393 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T21:56:00.394 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.2 2024-09-13T21:56:00.589 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:00 smithi154 systemd[1]: Stopping Ceph osd.2 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:56:01.089 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:00 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[76677]: 2024-09-13T21:56:00.733+0000 7f50db937640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:56:01.089 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:00 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[76677]: 2024-09-13T21:56:00.733+0000 7f50db937640 -1 osd.2 172 *** Got signal Terminated *** 2024-09-13T21:56:01.089 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:00 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2[76677]: 2024-09-13T21:56:00.733+0000 7f50db937640 -1 osd.2 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:56:06.452 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:06 smithi154 podman[99141]: 2024-09-13 21:56:06.178728112 +0000 UTC m=+5.568100555 container died cc7d439fad81d5083252cbfba8f22f3dfc1217d5f7d828fc9453860eccdbf16e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T21:56:06.827 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:06 smithi154 podman[99141]: 2024-09-13 21:56:06.541694892 +0000 UTC m=+5.931067344 container cleanup cc7d439fad81d5083252cbfba8f22f3dfc1217d5f7d828fc9453860eccdbf16e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-13T21:56:06.827 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:06 smithi154 bash[99141]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2 2024-09-13T21:56:07.089 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:06 smithi154 podman[99188]: 2024-09-13 21:56:06.82653258 +0000 UTC m=+0.650490365 container remove cc7d439fad81d5083252cbfba8f22f3dfc1217d5f7d828fc9453860eccdbf16e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be) 2024-09-13T21:56:07.464 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:07 smithi154 podman[99278]: 2024-09-13 21:56:07.064958771 +0000 UTC m=+0.016535389 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:56:07.464 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:07 smithi154 podman[99278]: 2024-09-13 21:56:07.185816983 +0000 UTC m=+0.137393592 container create 266d4292f2ebf61ae036e94b576873f5aeef8964b00cd308d84bf64831fb385f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T21:56:07.464 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:07 smithi154 podman[99278]: 2024-09-13 21:56:07.388274394 +0000 UTC m=+0.339851004 container init 266d4292f2ebf61ae036e94b576873f5aeef8964b00cd308d84bf64831fb385f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-13T21:56:07.464 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:07 smithi154 podman[99278]: 2024-09-13 21:56:07.391574976 +0000 UTC m=+0.343151590 container start 266d4292f2ebf61ae036e94b576873f5aeef8964b00cd308d84bf64831fb385f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T21:56:07.464 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:07 smithi154 podman[99278]: 2024-09-13 21:56:07.463411532 +0000 UTC m=+0.414988155 container attach 266d4292f2ebf61ae036e94b576873f5aeef8964b00cd308d84bf64831fb385f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T21:56:07.839 INFO:journalctl@ceph.osd.2.smithi154.stdout:Sep 13 21:56:07 smithi154 podman[99278]: 2024-09-13 21:56:07.584417312 +0000 UTC m=+0.535993920 container died 266d4292f2ebf61ae036e94b576873f5aeef8964b00cd308d84bf64831fb385f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T21:56:08.218 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T21:56:08.258 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:56:08.259 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T21:56:08.259 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T21:56:08.259 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.3 2024-09-13T21:56:08.588 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:08 smithi154 systemd[1]: Stopping Ceph osd.3 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:56:08.839 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:08 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:56:08.586+0000 7f8aef498640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:56:08.839 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:08 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:56:08.586+0000 7f8aef498640 -1 osd.3 172 *** Got signal Terminated *** 2024-09-13T21:56:08.839 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:08 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:56:08.586+0000 7f8aef498640 -1 osd.3 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:56:11.028 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:10 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:10.745+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:11.339 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:10 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:56:10.977+0000 7f8ae7aa6640 -1 osd.3 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:46.731370+0000 front 2024-09-13T21:55:46.731437+0000 (oldest deadline 2024-09-13T21:56:10.831437+0000) 2024-09-13T21:56:11.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:11 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:11.723+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:12.339 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:12 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:56:12.018+0000 7f8ae7aa6640 -1 osd.3 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:46.731370+0000 front 2024-09-13T21:55:46.731437+0000 (oldest deadline 2024-09-13T21:56:10.831437+0000) 2024-09-13T21:56:12.649 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:12.414+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:13.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:12.710+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:13.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:12 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:12.679+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:13.339 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:12 smithi154 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3[79382]: 2024-09-13T21:56:12.982+0000 7f8ae7aa6640 -1 osd.3 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:46.731370+0000 front 2024-09-13T21:55:46.731437+0000 (oldest deadline 2024-09-13T21:56:10.831437+0000) 2024-09-13T21:56:13.686 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:13 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:13.416+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:13.686 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:13 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:13.665+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:14.149 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:13 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:13.684+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:14.289 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:14 smithi154 podman[99381]: 2024-09-13 21:56:14.01796285 +0000 UTC m=+5.537213985 container died 287dcaef58cf28cfb35efcf116552d724635e73545be119b609fe99033535254 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T21:56:14.589 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:14 smithi154 podman[99381]: 2024-09-13 21:56:14.361649095 +0000 UTC m=+5.880900228 container cleanup 287dcaef58cf28cfb35efcf116552d724635e73545be119b609fe99033535254 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:56:14.589 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:14 smithi154 bash[99381]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3 2024-09-13T21:56:14.646 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:14 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:14.384+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:14.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:14 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:14.644+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:14.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:14 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:14.673+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:14.983 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:14 smithi154 podman[99401]: 2024-09-13 21:56:14.621155945 +0000 UTC m=+0.604991918 container remove 287dcaef58cf28cfb35efcf116552d724635e73545be119b609fe99033535254 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:56:15.283 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:14 smithi154 podman[99488]: 2024-09-13 21:56:14.886036741 +0000 UTC m=+0.016460837 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:56:15.284 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:15 smithi154 podman[99488]: 2024-09-13 21:56:15.01382295 +0000 UTC m=+0.144247101 container create 7810a8ff7cac8e5b50b1e72fe5fea1ac6abb017779cdd7063e39032ec102ed8b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True) 2024-09-13T21:56:15.284 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:15 smithi154 podman[99488]: 2024-09-13 21:56:15.215950937 +0000 UTC m=+0.346375033 container init 7810a8ff7cac8e5b50b1e72fe5fea1ac6abb017779cdd7063e39032ec102ed8b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-13T21:56:15.284 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:15 smithi154 podman[99488]: 2024-09-13 21:56:15.219173305 +0000 UTC m=+0.349597403 container start 7810a8ff7cac8e5b50b1e72fe5fea1ac6abb017779cdd7063e39032ec102ed8b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be) 2024-09-13T21:56:15.284 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:15 smithi154 podman[99488]: 2024-09-13 21:56:15.282610192 +0000 UTC m=+0.413034302 container attach 7810a8ff7cac8e5b50b1e72fe5fea1ac6abb017779cdd7063e39032ec102ed8b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:56:15.589 INFO:journalctl@ceph.osd.3.smithi154.stdout:Sep 13 21:56:15 smithi154 podman[99488]: 2024-09-13 21:56:15.407439265 +0000 UTC m=+0.537863361 container died 7810a8ff7cac8e5b50b1e72fe5fea1ac6abb017779cdd7063e39032ec102ed8b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T21:56:15.630 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:15 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:15.191+0000 7ff277052640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:49.218354+0000 front 2024-09-13T21:55:49.218380+0000 (oldest deadline 2024-09-13T21:56:14.518545+0000) 2024-09-13T21:56:15.631 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:15 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:15.381+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:15.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:15 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:15.629+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:15.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:15 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:15.691+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:15.966 DEBUG:teuthology.orchestra.run.smithi154:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T21:56:16.005 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:56:16.005 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T21:56:16.005 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T21:56:16.006 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.4 2024-09-13T21:56:16.316 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:16 smithi181 systemd[1]: Stopping Ceph osd.4 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:56:16.316 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:16 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:16.211+0000 7ff277052640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:49.218354+0000 front 2024-09-13T21:55:49.218380+0000 (oldest deadline 2024-09-13T21:56:14.518545+0000) 2024-09-13T21:56:16.316 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:16 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:16.314+0000 7ff27aa3c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:56:16.316 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:16 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:16.314+0000 7ff27aa3c640 -1 osd.4 172 *** Got signal Terminated *** 2024-09-13T21:56:16.316 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:16 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:16.314+0000 7ff27aa3c640 -1 osd.4 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:56:16.596 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:16 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:16.370+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:16.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:16 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:16.594+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:16.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:16 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:16.655+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:17.583 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:17 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:17.252+0000 7ff277052640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:49.218354+0000 front 2024-09-13T21:55:49.218380+0000 (oldest deadline 2024-09-13T21:56:14.518545+0000) 2024-09-13T21:56:17.583 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:17 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:17.396+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:17.899 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:17 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:17.581+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:17.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:17 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:17.636+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:18.574 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:18 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:18.213+0000 7ff277052640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:49.218354+0000 front 2024-09-13T21:55:49.218380+0000 (oldest deadline 2024-09-13T21:56:14.518545+0000) 2024-09-13T21:56:18.574 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:18 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:18.445+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:18.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:18 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:18.572+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:18.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:18 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:18.657+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:19.493 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:19 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:19.241+0000 7ff277052640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:49.218354+0000 front 2024-09-13T21:55:49.218380+0000 (oldest deadline 2024-09-13T21:56:14.518545+0000) 2024-09-13T21:56:19.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:19 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:19.537+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:19.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:19 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:19.654+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:19.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:19 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:19.492+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:20.539 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:20 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:20.285+0000 7ff277052640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:49.218354+0000 front 2024-09-13T21:55:49.218380+0000 (oldest deadline 2024-09-13T21:56:14.518545+0000) 2024-09-13T21:56:20.540 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:20 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:20.530+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:20.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:20 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:20.696+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:20.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:20 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:20.538+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:21.551 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:21.480+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:21.551 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:21.250+0000 7ff277052640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:49.218354+0000 front 2024-09-13T21:55:49.218380+0000 (oldest deadline 2024-09-13T21:56:14.518545+0000) 2024-09-13T21:56:21.551 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4[56535]: 2024-09-13T21:56:21.250+0000 7ff277052640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:54.519233+0000 front 2024-09-13T21:55:54.519144+0000 (oldest deadline 2024-09-13T21:56:20.319498+0000) 2024-09-13T21:56:21.900 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:21 smithi181 podman[73402]: 2024-09-13 21:56:21.707605801 +0000 UTC m=+5.503035653 container died e8ce19a28c45ae3b23987cb3c0f5873249f690f994c123e7e36fa78623bb3198 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-13T21:56:21.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:21.693+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:21.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:21.549+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:21.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:21 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:21.549+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:22.283 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:22 smithi181 podman[73402]: 2024-09-13 21:56:22.233511821 +0000 UTC m=+6.028941661 container cleanup e8ce19a28c45ae3b23987cb3c0f5873249f690f994c123e7e36fa78623bb3198 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-13T21:56:22.284 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:22 smithi181 bash[73402]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4 2024-09-13T21:56:22.555 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:22 smithi181 podman[73417]: 2024-09-13 21:56:22.40886091 +0000 UTC m=+0.701233899 container remove e8ce19a28c45ae3b23987cb3c0f5873249f690f994c123e7e36fa78623bb3198 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T21:56:22.555 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:22 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:22.448+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:22.849 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:22 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:22.700+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:22.849 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:22 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:22.700+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:22.849 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:22 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:22.553+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:22.850 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:22 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:22.553+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:23.150 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:22 smithi181 podman[73510]: 2024-09-13 21:56:22.751470643 +0000 UTC m=+0.015991990 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:56:23.150 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:22 smithi181 podman[73510]: 2024-09-13 21:56:22.859678763 +0000 UTC m=+0.124200139 container create 46b7b5a3b93690153af37598a8085eb0089acf630ec76c13bb9f69465ce57209 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T21:56:23.150 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:23 smithi181 podman[73510]: 2024-09-13 21:56:23.079324989 +0000 UTC m=+0.343846340 container init 46b7b5a3b93690153af37598a8085eb0089acf630ec76c13bb9f69465ce57209 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T21:56:23.150 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:23 smithi181 podman[73510]: 2024-09-13 21:56:23.083274877 +0000 UTC m=+0.347796212 container start 46b7b5a3b93690153af37598a8085eb0089acf630ec76c13bb9f69465ce57209 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T21:56:23.481 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:23 smithi181 podman[73510]: 2024-09-13 21:56:23.179186101 +0000 UTC m=+0.443707447 container attach 46b7b5a3b93690153af37598a8085eb0089acf630ec76c13bb9f69465ce57209 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T21:56:23.481 INFO:journalctl@ceph.osd.4.smithi181.stdout:Sep 13 21:56:23 smithi181 podman[73510]: 2024-09-13 21:56:23.282089258 +0000 UTC m=+0.546610593 container died 46b7b5a3b93690153af37598a8085eb0089acf630ec76c13bb9f69465ce57209 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-ecab7be, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-13T21:56:23.859 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:23 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:23.479+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:23.859 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:23 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:23.479+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.655341+0000 front 2024-09-13T21:55:55.655275+0000 (oldest deadline 2024-09-13T21:56:22.655730+0000) 2024-09-13T21:56:23.859 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:23 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:23.717+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:23.859 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:23 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:23.717+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:23.859 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:23 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:23.505+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:23.860 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:23 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:23.505+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:23.877 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T21:56:23.916 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:56:23.916 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T21:56:23.916 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T21:56:23.916 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.5 2024-09-13T21:56:24.149 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:23 smithi181 systemd[1]: Stopping Ceph osd.5 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:56:24.517 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:24.252+0000 7fae5909b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:56:24.517 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:24.252+0000 7fae5909b640 -1 osd.5 172 *** Got signal Terminated *** 2024-09-13T21:56:24.517 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:24.252+0000 7fae5909b640 -1 osd.5 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:56:24.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:24.518+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:24.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:24.518+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.655341+0000 front 2024-09-13T21:55:55.655275+0000 (oldest deadline 2024-09-13T21:56:22.655730+0000) 2024-09-13T21:56:24.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:24.680+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:24.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:24.680+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:24.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:24.516+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:24.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:24 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:24.516+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:25.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:25.485+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:25.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:25.485+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.655341+0000 front 2024-09-13T21:55:55.655275+0000 (oldest deadline 2024-09-13T21:56:22.655730+0000) 2024-09-13T21:56:25.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:25.696+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:25.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:25.696+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:25.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:25.551+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:25.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:25 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:25.551+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:26.899 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:26 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:26.484+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:26.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:26 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:26.484+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.655341+0000 front 2024-09-13T21:55:55.655275+0000 (oldest deadline 2024-09-13T21:56:22.655730+0000) 2024-09-13T21:56:26.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:26 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:26.723+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:26.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:26 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:26.723+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:26.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:26 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:26.541+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:26.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:26 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:26.541+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:27.774 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:27 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:27.511+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:27.775 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:27 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:27.511+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.655341+0000 front 2024-09-13T21:55:55.655275+0000 (oldest deadline 2024-09-13T21:56:22.655730+0000) 2024-09-13T21:56:27.775 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:27 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:27.538+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:27.775 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:27 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:27.538+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:27.775 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:27 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:27.538+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:28.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:27 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:27.773+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:28.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:27 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:27.773+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:28.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:27 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:27.773+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:28.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:28 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:28.505+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.252697+0000 front 2024-09-13T21:55:47.252670+0000 (oldest deadline 2024-09-13T21:56:10.653279+0000) 2024-09-13T21:56:28.900 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:28 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5[58795]: 2024-09-13T21:56:28.505+0000 7fae556b1640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.655341+0000 front 2024-09-13T21:55:55.655275+0000 (oldest deadline 2024-09-13T21:56:22.655730+0000) 2024-09-13T21:56:28.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:28 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:28.739+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:28.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:28 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:28.739+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:28.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:28 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:28.739+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:28.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:28 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:28.564+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:28.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:28 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:28.564+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:28.901 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:28 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:28.564+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:29.897 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:29 smithi181 podman[73654]: 2024-09-13 21:56:29.6312818 +0000 UTC m=+5.483240482 container died 80819532f2ce3c06e988b8fe7fb75cfa5c0e28e039f8353db7e8e1069c6b5227 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T21:56:29.897 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:29.547+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:29.897 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:29.547+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:29.897 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:29.547+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:29.898 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:29.780+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:29.898 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:29.780+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:29.898 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:29 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:29.780+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:30.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:29 smithi181 podman[73654]: 2024-09-13 21:56:29.969593678 +0000 UTC m=+5.821552351 container cleanup 80819532f2ce3c06e988b8fe7fb75cfa5c0e28e039f8353db7e8e1069c6b5227 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-ecab7be) 2024-09-13T21:56:30.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:29 smithi181 bash[73654]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5 2024-09-13T21:56:30.570 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:30 smithi181 podman[73670]: 2024-09-13 21:56:30.153974425 +0000 UTC m=+0.519443979 container remove 80819532f2ce3c06e988b8fe7fb75cfa5c0e28e039f8353db7e8e1069c6b5227 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-13T21:56:30.858 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:30 smithi181 podman[73759]: 2024-09-13 21:56:30.579912028 +0000 UTC m=+0.112936736 container create ee327713eb16520741f8e1012e21e17b3c38c28142bb2b201703fac9a00801c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T21:56:30.858 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:30 smithi181 podman[73759]: 2024-09-13 21:56:30.483302777 +0000 UTC m=+0.016327492 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:56:30.858 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:30 smithi181 podman[73759]: 2024-09-13 21:56:30.782490376 +0000 UTC m=+0.315515085 container init ee327713eb16520741f8e1012e21e17b3c38c28142bb2b201703fac9a00801c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T21:56:30.858 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:30 smithi181 podman[73759]: 2024-09-13 21:56:30.785733573 +0000 UTC m=+0.318758287 container start ee327713eb16520741f8e1012e21e17b3c38c28142bb2b201703fac9a00801c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:56:30.858 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:30 smithi181 podman[73759]: 2024-09-13 21:56:30.857328945 +0000 UTC m=+0.390353657 container attach ee327713eb16520741f8e1012e21e17b3c38c28142bb2b201703fac9a00801c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T21:56:30.859 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:30 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:30.760+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:30.859 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:30 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:30.760+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:30.859 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:30 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:30.760+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:30.859 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:30 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:30.569+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:30.859 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:30 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:30.569+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:30.859 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:30 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:30.569+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:31.150 INFO:journalctl@ceph.osd.5.smithi181.stdout:Sep 13 21:56:30 smithi181 podman[73759]: 2024-09-13 21:56:30.975728837 +0000 UTC m=+0.508753553 container died ee327713eb16520741f8e1012e21e17b3c38c28142bb2b201703fac9a00801c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-13T21:56:31.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T21:56:31.586 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:56:31.586 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T21:56:31.586 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T21:56:31.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.6 2024-09-13T21:56:31.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:31 smithi181 systemd[1]: Stopping Ceph osd.6 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:56:31.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:31.756+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:31.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:31.756+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:31.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:31.756+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:31.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:31.534+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:31.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:31.534+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:31.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:31.534+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:32.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:31.922+0000 7f4fdb754640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:56:32.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:31.922+0000 7f4fdb754640 -1 osd.6 172 *** Got signal Terminated *** 2024-09-13T21:56:32.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:31 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:31.922+0000 7f4fdb754640 -1 osd.6 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:56:32.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:32 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:32.511+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:32.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:32 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:32.511+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:32.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:32 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:32.511+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:32.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:32 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:32.732+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:32.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:32 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:32.732+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:32.900 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:32 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:32.732+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:33.759 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:33 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:33.468+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:33.760 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:33 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:33.468+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:33.760 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:33 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:33.468+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:34.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:33 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:33.758+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:34.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:33 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:33.758+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:34.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:33 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:33.758+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:34.799 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:34 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:34.432+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:34.799 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:34 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:34.432+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:34.799 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:34 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:34.432+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:35.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:34 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:34.798+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:35.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:34 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:34.798+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:35.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:34 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:34.798+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:35.649 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:35.385+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:35.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:35.385+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:35.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:35.385+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:36.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:35.754+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:36.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:35.754+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:36.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:35 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:35.754+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:36.649 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:36 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:36.366+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:36.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:36 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:36.366+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:36.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:36 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:36.366+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:37.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:36 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:36.775+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:47.871258+0000 front 2024-09-13T21:55:47.871188+0000 (oldest deadline 2024-09-13T21:56:12.571220+0000) 2024-09-13T21:56:37.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:36 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:36.775+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:57.772483+0000 front 2024-09-13T21:55:57.772632+0000 (oldest deadline 2024-09-13T21:56:22.472558+0000) 2024-09-13T21:56:37.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:36 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6[61038]: 2024-09-13T21:56:36.775+0000 7f4fd3d62640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:04.173447+0000 front 2024-09-13T21:56:04.173429+0000 (oldest deadline 2024-09-13T21:56:27.673338+0000) 2024-09-13T21:56:37.650 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:37 smithi181 podman[73863]: 2024-09-13 21:56:37.529708225 +0000 UTC m=+5.720863463 container died 07e99a632a762b8c01f5a11a40fa383f7717431e5055380be810e17bcf26728a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T21:56:37.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:37 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:37.358+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:37.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:37 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:37.358+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:37.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:37 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:37.358+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:38.148 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:37 smithi181 podman[73863]: 2024-09-13 21:56:37.897003678 +0000 UTC m=+6.088158915 container cleanup 07e99a632a762b8c01f5a11a40fa383f7717431e5055380be810e17bcf26728a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T21:56:38.148 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:37 smithi181 bash[73863]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6 2024-09-13T21:56:38.400 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:38 smithi181 podman[73878]: 2024-09-13 21:56:38.147948828 +0000 UTC m=+0.616209293 container remove 07e99a632a762b8c01f5a11a40fa383f7717431e5055380be810e17bcf26728a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T21:56:38.400 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:38 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:38.329+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:38.400 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:38 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:38.329+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:38.400 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:38 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:38.329+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:38.400 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:38 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:38.329+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6830 osd.3 since back 2024-09-13T21:56:13.457333+0000 front 2024-09-13T21:56:13.457359+0000 (oldest deadline 2024-09-13T21:56:38.157354+0000) 2024-09-13T21:56:38.819 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:38 smithi181 podman[73966]: 2024-09-13 21:56:38.379649485 +0000 UTC m=+0.015690999 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:56:38.819 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:38 smithi181 podman[73966]: 2024-09-13 21:56:38.499035113 +0000 UTC m=+0.135076620 container create 47ff1566400de60d198bb0dfff5f7752d49fcfbcc256fb08f64f7712439ed0c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-13T21:56:38.819 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:38 smithi181 podman[73966]: 2024-09-13 21:56:38.718259691 +0000 UTC m=+0.354301199 container init 47ff1566400de60d198bb0dfff5f7752d49fcfbcc256fb08f64f7712439ed0c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T21:56:38.819 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:38 smithi181 podman[73966]: 2024-09-13 21:56:38.721444022 +0000 UTC m=+0.357485533 container start 47ff1566400de60d198bb0dfff5f7752d49fcfbcc256fb08f64f7712439ed0c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T21:56:38.819 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:38 smithi181 podman[73966]: 2024-09-13 21:56:38.818204993 +0000 UTC m=+0.454246522 container attach 47ff1566400de60d198bb0dfff5f7752d49fcfbcc256fb08f64f7712439ed0c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-13T21:56:39.150 INFO:journalctl@ceph.osd.6.smithi181.stdout:Sep 13 21:56:38 smithi181 podman[73966]: 2024-09-13 21:56:38.914289605 +0000 UTC m=+0.550331122 container died 47ff1566400de60d198bb0dfff5f7752d49fcfbcc256fb08f64f7712439ed0c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T21:56:39.455 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:39 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:39.378+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:39.456 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:39 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:39.378+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:39.456 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:39 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:39.378+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:39.456 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:39 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:39.378+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6830 osd.3 since back 2024-09-13T21:56:13.457333+0000 front 2024-09-13T21:56:13.457359+0000 (oldest deadline 2024-09-13T21:56:38.157354+0000) 2024-09-13T21:56:39.481 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T21:56:39.519 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:56:39.520 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T21:56:39.520 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T21:56:39.520 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.7 2024-09-13T21:56:39.892 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:39 smithi181 systemd[1]: Stopping Ceph osd.7 for ad78d70a-7217-11ef-bceb-c7b262605968... 2024-09-13T21:56:40.149 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:39 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:39.891+0000 7efe294b6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T21:56:40.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:39 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:39.891+0000 7efe294b6640 -1 osd.7 172 *** Got signal Terminated *** 2024-09-13T21:56:40.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:39 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:39.891+0000 7efe294b6640 -1 osd.7 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T21:56:40.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:40 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:40.410+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:40.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:40 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:40.410+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:40.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:40 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:40.410+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:40.900 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:40 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:40.410+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6830 osd.3 since back 2024-09-13T21:56:13.457333+0000 front 2024-09-13T21:56:13.457359+0000 (oldest deadline 2024-09-13T21:56:38.157354+0000) 2024-09-13T21:56:41.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:41 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:41.363+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:41.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:41 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:41.363+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:41.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:41 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:41.363+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:41.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:41 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:41.363+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6830 osd.3 since back 2024-09-13T21:56:13.457333+0000 front 2024-09-13T21:56:13.457359+0000 (oldest deadline 2024-09-13T21:56:38.157354+0000) 2024-09-13T21:56:42.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:42 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:42.395+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:42.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:42 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:42.395+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:42.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:42 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:42.395+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:42.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:42 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:42.395+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6830 osd.3 since back 2024-09-13T21:56:13.457333+0000 front 2024-09-13T21:56:13.457359+0000 (oldest deadline 2024-09-13T21:56:38.157354+0000) 2024-09-13T21:56:43.833 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:43 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:43.417+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:43.833 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:43 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:43.417+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:43.833 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:43 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:43.417+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:43.833 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:43 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:43.417+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6830 osd.3 since back 2024-09-13T21:56:13.457333+0000 front 2024-09-13T21:56:13.457359+0000 (oldest deadline 2024-09-13T21:56:38.157354+0000) 2024-09-13T21:56:44.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:44 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:44.385+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6806 osd.0 since back 2024-09-13T21:55:44.853719+0000 front 2024-09-13T21:55:44.853674+0000 (oldest deadline 2024-09-13T21:56:11.853921+0000) 2024-09-13T21:56:44.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:44 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:44.385+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6814 osd.1 since back 2024-09-13T21:55:55.354948+0000 front 2024-09-13T21:55:55.354989+0000 (oldest deadline 2024-09-13T21:56:20.655007+0000) 2024-09-13T21:56:44.650 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:44 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:44.385+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6822 osd.2 since back 2024-09-13T21:56:02.356061+0000 front 2024-09-13T21:56:02.355939+0000 (oldest deadline 2024-09-13T21:56:27.055929+0000) 2024-09-13T21:56:44.651 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:44 smithi181 ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7[63331]: 2024-09-13T21:56:44.385+0000 7efe25acc640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.154:6830 osd.3 since back 2024-09-13T21:56:13.457333+0000 front 2024-09-13T21:56:13.457359+0000 (oldest deadline 2024-09-13T21:56:38.157354+0000) 2024-09-13T21:56:45.751 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:45 smithi181 podman[74068]: 2024-09-13 21:56:45.472915051 +0000 UTC m=+5.722889211 container died a55b1845224949f619dc4f5fd9aa189d8684326b7c1f530f9ac63e072da52c1a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-13T21:56:46.006 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:45 smithi181 podman[74068]: 2024-09-13 21:56:45.824475167 +0000 UTC m=+6.074449325 container cleanup a55b1845224949f619dc4f5fd9aa189d8684326b7c1f530f9ac63e072da52c1a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T21:56:46.006 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:45 smithi181 bash[74068]: ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7 2024-09-13T21:56:46.399 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:46 smithi181 podman[74127]: 2024-09-13 21:56:46.005684387 +0000 UTC m=+0.522442277 container remove a55b1845224949f619dc4f5fd9aa189d8684326b7c1f530f9ac63e072da52c1a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T21:56:46.756 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:46 smithi181 podman[74220]: 2024-09-13 21:56:46.33520796 +0000 UTC m=+0.015863428 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T21:56:46.756 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:46 smithi181 podman[74220]: 2024-09-13 21:56:46.447755641 +0000 UTC m=+0.128411102 container create 28b01ab36c5c451a712ff7f3488556d4f995afb588e527cd658ceabb5e8a678a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T21:56:46.757 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:46 smithi181 podman[74220]: 2024-09-13 21:56:46.671352593 +0000 UTC m=+0.352008055 container init 28b01ab36c5c451a712ff7f3488556d4f995afb588e527cd658ceabb5e8a678a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T21:56:46.757 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:46 smithi181 podman[74220]: 2024-09-13 21:56:46.674535592 +0000 UTC m=+0.355191055 container start 28b01ab36c5c451a712ff7f3488556d4f995afb588e527cd658ceabb5e8a678a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0) 2024-09-13T21:56:46.757 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:46 smithi181 podman[74220]: 2024-09-13 21:56:46.755975211 +0000 UTC m=+0.436630702 container attach 28b01ab36c5c451a712ff7f3488556d4f995afb588e527cd658ceabb5e8a678a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T21:56:47.150 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:46 smithi181 podman[74220]: 2024-09-13 21:56:46.862145649 +0000 UTC m=+0.542801119 container died 28b01ab36c5c451a712ff7f3488556d4f995afb588e527cd658ceabb5e8a678a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-13T21:56:47.407 INFO:journalctl@ceph.osd.7.smithi181.stdout:Sep 13 21:56:47 smithi181 podman[74243]: 2024-09-13 21:56:47.390013736 +0000 UTC m=+0.521392496 container remove 28b01ab36c5c451a712ff7f3488556d4f995afb588e527cd658ceabb5e8a678a (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-ad78d70a-7217-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be) 2024-09-13T21:56:47.413 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ad78d70a-7217-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T21:56:47.452 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T21:56:47.452 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T21:56:47.453 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ad78d70a-7217-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T21:59:44.341 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ad78d70a-7217-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T22:00:03.947 DEBUG:teuthology.orchestra.run.smithi154:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:00:03.982 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:00:04.009 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T22:00:04.011 DEBUG:teuthology.misc:Transferring archived files from smithi154:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199/remote/smithi154/crash 2024-09-13T22:00:04.012 DEBUG:teuthology.orchestra.run.smithi154:> sudo tar c -f - -C /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/crash -- . 2024-09-13T22:00:04.103 INFO:teuthology.orchestra.run.smithi154.stderr:tar: /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T22:00:04.104 INFO:teuthology.orchestra.run.smithi154.stderr:tar: Error is not recoverable: exiting now 2024-09-13T22:00:04.106 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199/remote/smithi181/crash 2024-09-13T22:00:04.106 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/crash -- . 2024-09-13T22:00:04.146 INFO:teuthology.orchestra.run.smithi181.stderr:tar: /var/lib/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T22:00:04.146 INFO:teuthology.orchestra.run.smithi181.stderr:tar: Error is not recoverable: exiting now 2024-09-13T22:00:04.147 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T22:00:04.147 DEBUG:teuthology.orchestra.run.smithi154:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-13T22:00:04.186 INFO:tasks.cephadm:Compressing logs... 2024-09-13T22:00:04.186 DEBUG:teuthology.orchestra.run.smithi154:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T22:00:04.228 DEBUG:teuthology.orchestra.run.smithi181:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T22:00:04.253 INFO:teuthology.orchestra.run.smithi154.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T22:00:04.253 INFO:teuthology.orchestra.run.smithi154.stderr:: No such file or directory 2024-09-13T22:00:04.257 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T22:00:04.257 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mon.smithi154.log 2024-09-13T22:00:04.258 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T22:00:04.258 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mon.smithi154.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T22:00:04.258 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T22:00:04.258 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-volume.log: /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T22:00:04.258 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.log 2024-09-13T22:00:04.258 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.2.log: 85.8% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T22:00:04.259 INFO:teuthology.orchestra.run.smithi154.stderr: 91.6% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T22:00:04.259 INFO:teuthology.orchestra.run.smithi181.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi181.mgvpzf.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:: No such file or directory 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi181.mgvpzf.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr: 88.3% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi181.mgvpzf.log.gz 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.log 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.5.log: 85.8% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T22:00:04.260 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mgr.smithi181.byvkib.log 2024-09-13T22:00:04.261 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.log: 91.7% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T22:00:04.262 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mgr.smithi154.ayrwdi.log 2024-09-13T22:00:04.262 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T22:00:04.263 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mgr.smithi154.ayrwdi.log: gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi154.uynwpr.log 2024-09-13T22:00:04.263 INFO:teuthology.orchestra.run.smithi181.stderr: 87.0% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T22:00:04.263 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mon.smithi181.log 2024-09-13T22:00:04.263 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mgr.smithi181.byvkib.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T22:00:04.265 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.3.log: 87.0% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T22:00:04.268 INFO:teuthology.orchestra.run.smithi154.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T22:00:04.269 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi154.uynwpr.log: 91.2% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T22:00:04.269 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mon.smithi181.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T22:00:04.269 INFO:teuthology.orchestra.run.smithi181.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T22:00:04.287 INFO:teuthology.orchestra.run.smithi154.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi154.uynwpr.log.gz 2024-09-13T22:00:04.308 INFO:teuthology.orchestra.run.smithi154.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T22:00:04.309 INFO:teuthology.orchestra.run.smithi181.stderr: 89.9% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mgr.smithi181.byvkib.log.gz 2024-09-13T22:00:04.603 INFO:teuthology.orchestra.run.smithi181.stderr: 92.6% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mon.smithi181.log.gz 2024-09-13T22:00:04.634 INFO:teuthology.orchestra.run.smithi154.stderr: 88.7% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mgr.smithi154.ayrwdi.log.gz 2024-09-13T22:00:06.151 INFO:teuthology.orchestra.run.smithi154.stderr: 91.9% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-mon.smithi154.log.gz 2024-09-13T22:00:18.028 INFO:teuthology.orchestra.run.smithi154.stderr: 94.4% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T22:00:18.637 INFO:teuthology.orchestra.run.smithi181.stderr: 93.8% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T22:00:20.214 INFO:teuthology.orchestra.run.smithi181.stderr: 94.2% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T22:00:21.294 INFO:teuthology.orchestra.run.smithi154.stderr: 94.1% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T22:00:21.976 INFO:teuthology.orchestra.run.smithi154.stderr: 93.9% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T22:00:23.100 INFO:teuthology.orchestra.run.smithi154.stderr: 94.0% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T22:00:23.102 INFO:teuthology.orchestra.run.smithi154.stderr: 2024-09-13T22:00:23.102 INFO:teuthology.orchestra.run.smithi154.stderr:real 0m18.860s 2024-09-13T22:00:23.102 INFO:teuthology.orchestra.run.smithi154.stderr:user 1m6.809s 2024-09-13T22:00:23.102 INFO:teuthology.orchestra.run.smithi154.stderr:sys 0m2.750s 2024-09-13T22:00:24.014 INFO:teuthology.orchestra.run.smithi181.stderr: 94.1% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T22:00:27.130 INFO:teuthology.orchestra.run.smithi181.stderr: 94.0% -- replaced with /var/log/ceph/ad78d70a-7217-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T22:00:27.132 INFO:teuthology.orchestra.run.smithi181.stderr: 2024-09-13T22:00:27.132 INFO:teuthology.orchestra.run.smithi181.stderr:real 0m22.888s 2024-09-13T22:00:27.132 INFO:teuthology.orchestra.run.smithi181.stderr:user 1m10.433s 2024-09-13T22:00:27.132 INFO:teuthology.orchestra.run.smithi181.stderr:sys 0m2.840s 2024-09-13T22:00:27.132 INFO:tasks.cephadm:Archiving logs... 2024-09-13T22:00:27.133 DEBUG:teuthology.misc:Transferring archived files from smithi154:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199/remote/smithi154/log 2024-09-13T22:00:27.134 DEBUG:teuthology.orchestra.run.smithi154:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T22:00:36.464 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199/remote/smithi181/log 2024-09-13T22:00:36.465 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T22:00:46.260 INFO:tasks.cephadm:Removing cluster... 2024-09-13T22:00:46.260 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ad78d70a-7217-11ef-bceb-c7b262605968 --force 2024-09-13T22:00:46.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ad78d70a-7217-11ef-bceb-c7b262605968 --force 2024-09-13T22:00:47.006 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T22:00:47.006 DEBUG:teuthology.orchestra.run.smithi154:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T22:00:47.023 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T22:00:47.039 INFO:tasks.cephadm:Teardown complete 2024-09-13T22:00:47.039 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T22:00:47.050 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T22:00:47.051 DEBUG:teuthology.orchestra.run.smithi154:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:00:47.066 DEBUG:teuthology.orchestra.run.smithi181:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:00:47.078 INFO:teuthology.orchestra.run.smithi154.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:00:47.083 INFO:teuthology.orchestra.run.smithi154.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:00:47.083 INFO:teuthology.orchestra.run.smithi154.stdout:=============================================================================== 2024-09-13T22:00:47.083 INFO:teuthology.orchestra.run.smithi154.stdout:^* hv01.front.sepia.ceph.com 3 6 377 9 -230us[ -184us] +/- 49ms 2024-09-13T22:00:47.083 INFO:teuthology.orchestra.run.smithi154.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 11 +609us[ +655us] +/- 64ms 2024-09-13T22:00:47.083 INFO:teuthology.orchestra.run.smithi154.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 11 -235us[ -189us] +/- 69ms 2024-09-13T22:00:47.084 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:00:47.092 INFO:teuthology.orchestra.run.smithi181.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:00:47.096 INFO:teuthology.orchestra.run.smithi181.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:00:47.097 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================== 2024-09-13T22:00:47.097 INFO:teuthology.orchestra.run.smithi181.stdout:^* hv01.front.sepia.ceph.com 3 6 377 8 -220us[ -200us] +/- 49ms 2024-09-13T22:00:47.097 INFO:teuthology.orchestra.run.smithi181.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 12 +596us[ +616us] +/- 64ms 2024-09-13T22:00:47.097 INFO:teuthology.orchestra.run.smithi181.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 12 -246us[ -226us] +/- 69ms 2024-09-13T22:00:47.097 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:00:47.098 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T22:00:47.108 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T22:00:47.109 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T22:00:47.118 DEBUG:teuthology.orchestra.run.smithi154:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T22:00:47.204 DEBUG:teuthology.orchestra.run.smithi181:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T22:00:47.279 DEBUG:teuthology.orchestra.run.smithi154:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T22:00:47.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:00:47.308 DEBUG:teuthology.orchestra.run.smithi181:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T22:00:47.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:00:47.336 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T22:00:47.346 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T22:00:47.356 INFO:teuthology.task.internal:Duration was 1929.227270 seconds 2024-09-13T22:00:47.356 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T22:00:47.365 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T22:00:47.365 DEBUG:teuthology.orchestra.run.smithi154:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T22:00:47.369 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T22:00:47.407 INFO:teuthology.orchestra.run.smithi154.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:00:47.415 INFO:teuthology.orchestra.run.smithi181.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:00:47.876 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T22:00:47.876 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi154.front.sepia.ceph.com 2024-09-13T22:00:47.876 DEBUG:teuthology.orchestra.run.smithi154:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T22:00:47.938 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi181.front.sepia.ceph.com 2024-09-13T22:00:47.938 DEBUG:teuthology.orchestra.run.smithi181:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T22:00:47.977 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T22:00:47.977 DEBUG:teuthology.orchestra.run.smithi154:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T22:00:47.980 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T22:00:48.044 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T22:00:48.044 DEBUG:teuthology.orchestra.run.smithi154:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T22:00:48.047 DEBUG:teuthology.orchestra.run.smithi181:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T22:01:01.881 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T22:01:01.898 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T22:01:01.899 DEBUG:teuthology.orchestra.run.smithi154:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T22:01:01.925 DEBUG:teuthology.orchestra.run.smithi181:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T22:01:01.957 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T22:01:01.968 DEBUG:teuthology.orchestra.run.smithi154:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:01:01.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:01:01.991 INFO:teuthology.orchestra.run.smithi154.stdout:kernel.core_pattern = core 2024-09-13T22:01:02.024 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = core 2024-09-13T22:01:02.052 DEBUG:teuthology.orchestra.run.smithi154:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:01:02.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:01:02.086 DEBUG:teuthology.orchestra.run.smithi181:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:01:02.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:01:02.109 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T22:01:02.119 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T22:01:02.120 DEBUG:teuthology.misc:Transferring archived files from smithi154:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199/remote/smithi154 2024-09-13T22:01:02.121 DEBUG:teuthology.orchestra.run.smithi154:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T22:01:02.198 DEBUG:teuthology.misc:Transferring archived files from smithi181:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904199/remote/smithi181 2024-09-13T22:01:02.198 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T22:01:02.242 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T22:01:02.242 DEBUG:teuthology.orchestra.run.smithi154:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T22:01:02.244 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T22:01:02.292 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T22:01:02.304 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T22:01:02.304 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T22:01:02.314 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T22:01:02.315 DEBUG:teuthology.orchestra.run.smithi154:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T22:01:02.317 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T22:01:02.330 INFO:teuthology.orchestra.run.smithi154.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 22:01 /home/ubuntu/cephtest 2024-09-13T22:01:02.348 INFO:teuthology.orchestra.run.smithi181.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 22:01 /home/ubuntu/cephtest 2024-09-13T22:01:02.349 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T22:01:02.360 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T22:01:02.388 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1929.2272703647614 owner: scheduled_teuthology@teuthology success: true 2024-09-13T22:01:02.388 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T22:01:02.496 INFO:teuthology.run:pass